listpick 0.1.16.10__py3-none-any.whl → 0.1.16.12__py3-none-any.whl

listpick/listpick_app.py

@@ -18,9 +18,13 @@ from wcwidth import wcswidth
 from typing import Callable, Optional, Tuple, Dict
 import json
 import threading
+import multiprocessing
 import string
 import logging
-from copy import copy
+import copy
+import tempfile
+import queue
+from listpick.utils.generate_data_utils import ProcessSafePriorityQueue
 
 from listpick.pane.pane_utils import get_file_attributes
 from listpick.pane.left_pane_functions import *
@@ -70,9 +74,9 @@ class Picker:
         auto_refresh: bool =False,
         timer: float = 5,
 
-        get_new_data: bool =False,                          # Whether we can get new data
-        refresh_function: Optional[Callable] = lambda items, header, visible_rows_indices, getting_data: None,  # The function with which we get new data
-        get_data_startup: bool =False,                      # Whether we should get data at statrup
+        get_new_data: bool =False,
+        refresh_function: Optional[Callable] = lambda items, header, visible_rows_indices, getting_data: None,
+        get_data_startup: bool =False,
         track_entries_upon_refresh: bool = True,
         pin_cursor: bool = False,
         id_column: int = 0,
@@ -95,8 +99,12 @@ class Picker:
         user_opts : str = "",
         options_list: list[str] = [],
         user_settings : str = "",
+
         separator : str = "    ",
         header_separator : str = "   │",
+        header_separator_before_selected_column : str = "   ▐",
+
+        
         search_query : str = "",
         search_count : int = 0,
         search_index : int = 0,
@@ -111,6 +119,10 @@ class Picker:
         highlight_full_row: bool =False,
         crosshair_cursor: bool = False,
         cell_cursor: bool = True,
+        selected_char: str = "",
+        unselected_char: str = "",
+        selecting_char: str = "",
+        deselecting_char: str = "",
 
         items_per_page : int = -1,
         sort_method : int = 0,
@@ -197,6 +209,8 @@ class Picker:
         left_pane_index: int = 0,
 
         screen_size_function = lambda stdscr: os.get_terminal_size()[::-1],
+        generate_data_for_hidden_columns: bool = False,
+
 
         # getting_data: threading.Event = threading.Event(),
 
@@ -245,6 +259,7 @@ class Picker:
         self.user_settings = user_settings
         self.separator = separator
         self.header_separator = header_separator
+        self.header_separator_before_selected_column = header_separator_before_selected_column
         self.search_query = search_query
         self.search_count = search_count
         self.search_index = search_index
@@ -259,6 +274,10 @@ class Picker:
         self.highlight_full_row = highlight_full_row
         self.crosshair_cursor = crosshair_cursor
         self.cell_cursor = cell_cursor
+        self.selected_char = selected_char
+        self.unselected_char = unselected_char
+        self.selecting_char = selecting_char
+        self.deselecting_char = deselecting_char
 
         self.items_per_page = items_per_page
         self.sort_method = sort_method
@@ -361,8 +380,18 @@ class Picker:
 
         self.visible_rows_indices = []
 
+        self.generate_data_for_hidden_columns = generate_data_for_hidden_columns
+        self.thread_stop_event = threading.Event()
+        self.data_generation_queue = queue.PriorityQueue()
+        self.threads = []
 
 
+        self.process_manager = multiprocessing.Manager()
+        # self.data_generation_queue = ProcessSafePriorityQueue
+        self.processes = []
+        self.items_sync_loop_event = threading.Event() 
+        self.items_sync_thread = None
+
 
         self.initialise_picker_state(reset_colours=self.reset_colours)
 
@@ -378,6 +407,11 @@ class Picker:
         self.getting_data.set()
 
     def __sizeof__(self):
+        """
+        Return the approximate memory footprint of the Picker instance.
+
+        This includes the size of the instance itself and the sizes of its attributes.
+        """
 
         size = super().__sizeof__()
         
@@ -388,7 +422,19 @@ class Picker:
         return size
     
     def set_config(self, path: str ="~/.config/listpick/config.toml") -> bool:
-        """ Set config from toml file. """
+        """ Set config from toml file.
+
+        This method reads a configuration file in TOML format, applies settings 
+        to the Picker, and returns a boolean indicating success or failure.
+
+        Args:
+            path (str): The path to the configuration file.
+
+        Returns:
+            bool: True if the configuration was successfully set; False otherwise.
+        """
+        self.logger.info(f"function: set_config()")
+
         path = os.path.expanduser(os.path.expandvars(path))
         if not os.path.exists(path):
             return False
@@ -397,7 +443,9 @@ class Picker:
         except Exception as e:
             self.logger.error(f"get_config({path}) load error. {e}")
             return False
+        
 
+        # Change the global theme if colour_theme_number is in the loaded config
         if "general" in config:
             if "colour_theme_number" in config["general"] and config["general"]["colour_theme_number"] != self.colour_theme_number:
                 global COLOURS_SET 
@@ -405,7 +453,7 @@ class Picker:
                 self.colours_end = set_colours(pick=config["general"]["colour_theme_number"], start=1)
                 self.colours = get_colours(config["general"]["colour_theme_number"])
 
-        self.logger.info(f"function: set_config()")
+        # load the rest of the config options
         if "general" in config:
             for key, val in config["general"].items():
                 self.logger.info(f"set_config: key={key}, val={val}.")
@@ -417,30 +465,58 @@ class Picker:
         return True
 
     def get_config(self, path: str ="~/.config/listpick/config.toml") -> dict:
-        """ Get config from file. """
+        """ 
+        Retrieve configuration settings from a specified TOML file.
+
+        Args:
+            path (str): The file path of the configuration file. Default is 
+            ~/.config/listpick/config.toml.
+
+        Returns:
+            dict: A dictionary containing the configuration settings loaded
+            from the TOML file. In case of an error, an empty dictionary is returned.
+        """
+
         self.logger.info(f"function: get_config()")
         import toml
         with open(os.path.expanduser(path), "r") as f:
             config = toml.load(f)
             return config
 
-    def update_term_size(self):
+    def update_term_size(self) -> None:
+        """
+        Update self.term_h, self.term_w the function provided to the Picker.
+
+        Returns:
+            None
+        """
         self.term_h, self.term_w = self.screen_size_function(self.stdscr)
         # self.term_h, self.term_w = self.stdscr.getmaxyx()
         # self.term_w, self.term_h = os.get_terminal_size()
 
 
-    def get_term_size(self):
-        return self.stdscr.getmaxyx()
-        w, h = os.get_terminal_size()
-        return h, w
+    def get_term_size(self) -> Tuple[int, int]:
+        """
+        Get the current terminal size using the function provided to the Picker.
+
+        Returns:
+            Tuple[int, int]: A tuple containing the (height, width) of the terminal.
+        """
+        return self.screen_size_function(self.stdscr)
+        # return self.stdscr.getmaxyx()
+        # w, h = os.get_terminal_size()
+        # return h, w
 
-    def calculate_section_sizes(self):
+    def calculate_section_sizes(self) -> None:
         """
         Calculte the following for the Picker:
         self.items_per_page: the number of entry rows displayed
         self.bottom_space: the size of the footer + the bottom buffer space
         self.top_space: the size of the space at the top of the picker: title + modes + header + top_gap
+        Calculate and update the sizes of various sections of the Picker.
+
+        Returns:
+            None
         """
 
         self.logger.debug(f"function: calculate_section_sizes()")
@@ -494,8 +570,8 @@ class Picker:
             self.top_space += ((self.term_h-(self.top_space+self.bottom_space))-len(self.indexed_items))//2
 
         # self.column_widths
-        visible_column_widths = [c for i,c in enumerate(self.column_widths) if i not in self.hidden_columns]
-        visible_columns_total_width = sum(visible_column_widths) + len(self.separator)*(len(visible_column_widths)-1)
+        self.visible_column_widths = [c for i,c in enumerate(self.column_widths) if i not in self.hidden_columns]
+        visible_columns_total_width = sum(self.visible_column_widths) + len(self.separator)*(len(self.visible_column_widths)-1)
 
         # self.startx
         self.startx = 1 if self.highlight_full_row else 2
@@ -512,16 +588,27 @@ class Picker:
 
 
     def get_visible_rows(self) -> list[list[str]]:
+        """
+        Calculate and return the currently visible rows based on the cursor position and pagination settings.
 
+        This method determines which rows from the indexed items are visible on the screen,
+        accounting for pagination and scrolling. It sets the starting and ending indices
+        based on the current cursor position and the number of items per page.
+
+        Returns:
+            list[list[str]]: The currently visible rows as a list of lists, where each inner
+            list represents a row of data. If there are no indexed items, it returns the
+            items array.
+        """
         self.logger.debug(f"function: get_visible_rows()")
         ## Scroll with column select
         if self.paginate:
-            start_index = (self.cursor_pos//self.items_per_page) * self.items_per_page
+            start_index = (self.cursor_pos // self.items_per_page) * self.items_per_page
             end_index = min(start_index + self.items_per_page, len(self.indexed_items))
         ## Scroll
         else:
-            scrolloff = self.items_per_page//2
-            start_index = max(0, min(self.cursor_pos - (self.items_per_page-scrolloff), len(self.indexed_items)-self.items_per_page))
+            scrolloff = self.items_per_page // 2
+            start_index = max(0, min(self.cursor_pos - (self.items_per_page - scrolloff), len(self.indexed_items) - self.items_per_page))
             end_index = min(start_index + self.items_per_page, len(self.indexed_items))
         if len(self.indexed_items) == 0: start_index, end_index = 0, 0
 
@@ -534,13 +621,15 @@ class Picker:
     def initialise_picker_state(self, reset_colours=False) -> None:
         """ Initialise state variables for the picker. These are: debugging and colours. """
 
+        # Define global curses colours
         if curses.has_colors() and self.colours != None:
-            # raise Exception("Terminal does not support color")
             curses.start_color()
+
             if reset_colours:
                 global COLOURS_SET
                 COLOURS_SET = False
                 self.colours_end = set_colours(pick=self.colour_theme_number, start=self.colours_start)
+
             if curses.COLORS >= 255 and curses.COLOR_PAIRS >= 150:
                 self.colours_start = self.colours_start
                 self.notification_colours_start = self.colours_start+50
@@ -557,64 +646,40 @@ class Picker:
         self.colours = get_colours(self.colour_theme_number)
 
 
+        # Start logger
         debug_levels = [logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL]
         dbglvl =  debug_levels[self.debug_level]
         self.logger = setup_logger(name="picker_log", log_file="picker.log", log_enabled=self.debug, level =dbglvl)
         self.logger.info(f"Initialiasing Picker.")
+
         self.update_term_size()
-        # self.notification(self.stdscr, message=repr(self.logger))
-
-
-        #                     1              2             3                4              5
-        # logger = logging.getLogger(__file__)
-        # if self.debug_level == 0:
-        #     logger = logging.getLogger()
-        #     logger.disabled = True
-        # else:
-        #
-        #     file_handler = logging.FileHandler(f"{self.title}.log", mode='w')
-        #     formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s', '%m-%d-%Y %H:%M:%S')
-        #     file_handler.setFormatter(formatter)
-        #     logger.addHandler(file_handler)
-        #     logger.setLevel(debug_levels[self.debug_level-1])
-
-            # logging.basicConfig(
-            #     level=debug_levels[self.debug_level-1],
-            #     format='%(asctime)s - %(levelname)s - %(message)s',
-            #     datefmt='%m-%d-%Y %H:%M:%S',
-            #     filename=f"{self.title}.log",
-            #     filemode="w",
-            # )
-        #
-        # self.logger.info(f"Starging log. Log level {logger.getEffectiveLevel()}")
-        # self.logger.info(f"Starging log. Log level {repr(debug_levels)}, {self.debug_level}, {debug_levels[self.debug_level-1]}")
-        # self.notification(self.stdscr, f"Starging log. Log level {repr(debug_levels)}, {self.debug_level}, {debug_levels[self.debug_level-1]}")
-        # self.notification(self.stdscr, f"{__file__}")
-
-        ## Logging level plan
-        # DEBUG: loop functions, draw screen, etc.
-        # INFO: main functions 
-        # WARNING: any try-except fails 
-
-        # No set_escdelay function on windows.
+
+        # The curses implementation for some systems (e.g., windows) does not allow set_escdelay
         try:
             curses.set_escdelay(25)
         except:
             logging.warning("Error trying to set curses.set_escdelay")
 
-        # self.stdscr.clear()
-        # self.stdscr.refresh()
-        # self.draw_screen()
-
     def initialise_variables(self, get_data: bool = False) -> None:
-        """ Initialise the variables that keep track of the data. """
+        """
+        This method sets up the internal state of the Picker by initialising various attributes,
+        getting new data (if get_data is True), and ensuring that the lists used for tracking 
+        selections, options, and items are correctly of the correct type, size, and shape. If 
+        filter or sort queries are set then they are applied (or re-applied as the case may be).
+        The cursor_pos and selections are retained by tracking the id of the rows (where the id 
+        is row[self.id_column]). 
+
+        Parameters:
+        - get_data (bool): If True, pulls data synchronously and updates tracking variables.
+        """
 
         self.logger.info(f"function: initialise_variables()")
 
         tracking = False
 
         ## Get data synchronously
-        if get_data and self.refresh_function != None:
+        if get_data:
+            # Track cursor_pos and selections by ther id (row[self.id_column][col])
             if self.track_entries_upon_refresh and len(self.items) > 0:
                 tracking = True
                 selected_indices = get_selected_indices(self.selections)
@@ -622,34 +687,41 @@ class Picker:
                 self.ids = [item[self.id_column] for i, item in enumerate(self.items) if i in selected_indices]
                 self.ids_tuples = [(i, item[self.id_column]) for i, item in enumerate(self.items) if i in selected_indices]
         
-                if len(self.indexed_items) > 0 and len(self.indexed_items) >= self.cursor_pos and len(self.indexed_items[0][1]) >= self.id_column:
+                if len(self.indexed_items) > 0 and self.cursor_pos < len(self.indexed_items) and len(self.indexed_items[0][1]) >= self.id_column:
                     self.cursor_pos_id = self.indexed_items[self.cursor_pos][1][self.id_column]
                     self.cursor_pos_prev = self.cursor_pos
-        
-
 
+            # Set the state of the threading event
+            # Though we are getting data synchronously, we ensure the correct state for self.getting_data 
             self.getting_data.clear()
-            self.refresh_function(self.items, self.header, self.visible_rows_indices, self.getting_data)
+            self.refresh_function(
+                self.items,
+                self.header,
+                self.visible_rows_indices,
+                self.getting_data,
+                self.get_function_data(),
+            )
 
-        self.items = pad_lists_to_same_length(self.items)
                     
+        # Ensure that an emtpy items object has the form [[]]
         if self.items == []: self.items = [[]]
-        ## Ensure that items is a List[List[Str]] object
+
+        # Ensure that items is a List[List[Str]] object
         if len(self.items) > 0 and not isinstance(self.items[0], list):
             self.items = [[item] for item in self.items]
         # self.items = [[str(cell) for cell in row] for row in self.items]
 
+        # Ensure that the each of the rows of the items are of the same length
+        self.items = pad_lists_to_same_length(self.items)
 
         # Ensure that header is of the same length as the rows
         if self.header and len(self.items) > 0 and len(self.header) != len(self.items[0]):
             self.header = [str(self.header[i]) if i < len(self.header) else "" for i in range(len(self.items[0]))]
 
-        # Constants
-        # DEFAULT_ITEMS_PER_PAGE = os.get_terminal_size().lines - top_gap*2-2-int(bool(header))
-
         self.calculate_section_sizes()
 
-        # Initial states
+
+        # Ensure that the selection-tracking variables are the correct shape
         if len(self.selections) != len(self.items):
             self.selections = {i : False if i not in self.selections else bool(self.selections[i]) for i in range(len(self.items))}
 
@@ -660,31 +732,34 @@ class Picker:
             self.cell_selections = {}
             self.selected_cells_by_row = {}
 
+        def extend_list_to_length(lst, length, default_value):
+            """Extend a list to the target length using a default value."""
+            if len(lst) < length:
+                lst.extend([copy.deepcopy(default_value) for _ in range(length - len(lst))])
         
 
-        if len(self.require_option) < len(self.items):
-            self.require_option += [self.require_option_default for i in range(len(self.items)-len(self.require_option))]
-        if len(self.option_functions) < len(self.items):
-            self.option_functions += [self.default_option_function for i in range(len(self.items)-len(self.option_functions))]
-        if len(self.items)>0 and len(self.columns_sort_method) < len(self.items[0]):
-            self.columns_sort_method = self.columns_sort_method + [0 for i in range(len(self.items[0])-len(self.columns_sort_method))]
-        if len(self.items)>0 and len(self.sort_reverse) < len(self.items[0]):
-            self.sort_reverse = self.sort_reverse + [False for i in range(len(self.items[0])-len(self.sort_reverse))]
-        if len(self.items)>0 and len(self.editable_columns) < len(self.items[0]):
-            self.editable_columns = self.editable_columns + [self.editable_by_default for i in range(len(self.items[0])-len(self.editable_columns))]
-        if len(self.items)>0 and len(self.column_indices) < len(self.items[0]):
+        row_count = len(self.items)
+        col_count = len(self.items[0]) if row_count else 0
+
+        # Ensure that the length of the option lists are of the correct length.
+        if row_count > 0:
+            extend_list_to_length(self.require_option, length=row_count, default_value=self.require_option_default)
+            extend_list_to_length(self.option_functions, length=row_count, default_value=self.default_option_function)
+            extend_list_to_length(self.columns_sort_method, length=col_count, default_value=0)
+            extend_list_to_length(self.sort_reverse, length=col_count, default_value=False)
+            extend_list_to_length(self.editable_columns, length=col_count, default_value=self.editable_by_default)
+
+        if row_count > 0 and len(self.column_indices) < len(self.items[0]):
             self.column_indices = self.column_indices + [i for i in range(len(self.column_indices), len(self.items[0]))]
 
 
         
-        # items2 = [[row[self.column_indices[i]] for i in range(len(row))] for row in self.items]
-        # self.indexed_items = list(enumerate(items2))
+        # Create an indexed list of the items which will track the visible rows
         if self.items == [[]]: self.indexed_items = []
         else: self.indexed_items = list(enumerate(self.items))
 
-        # If a filter is passed then refilter
+        # Apply the filter query
         if self.filter_query:
-            # prev_index = self.indexed_items[cursor_pos][0] if len(self.indexed_items)>0 else 0
             # prev_index = self.indexed_items[cursor_pos][0] if len(self.indexed_items)>0 else 0
             self.indexed_items = filter_items(self.items, self.indexed_items, self.filter_query)
             if self.cursor_pos in [x[0] for x in self.indexed_items]: self.cursor_pos = [x[0] for x in self.indexed_items].index(self.cursor_pos)
@@ -700,22 +775,30 @@ class Picker:
             )
             if return_val:
                 self.cursor_pos, self.search_index, self.search_count, self.highlights = tmp_cursor, tmp_index, tmp_count, tmp_highlights
-        # If a sort is passed
+
+        # Apply the current sort method
         if len(self.indexed_items) > 0:
             sort_items(self.indexed_items, sort_method=self.columns_sort_method[self.sort_column], sort_column=self.sort_column, sort_reverse=self.sort_reverse[self.sort_column])  # Re-sort self.items based on new column
 
 
-        # Adjust variables to ensure correctness if errors
-        ## Move to a selectable row (if applicable)
+        # If we have more unselectable indices than rows, clear the unselectable_indices
         if len(self.items) <= len(self.unselectable_indices): self.unselectable_indices = []
-        new_pos = (self.cursor_pos)%len(self.items)
-        while new_pos in self.unselectable_indices and new_pos != self.cursor_pos:
-            new_pos = (new_pos + 1) % len(self.items)
 
-        assert new_pos < len(self.items)
-        self.cursor_pos = new_pos
+        # Move cursur to a selectable row if we are currently on an unselectable row)
+        if self.cursor_pos * len(self.items) in self.unselectable_indices:
+            original_pos = new_pos = (self.cursor_pos)%len(self.items)
+            while new_pos in self.unselectable_indices:
+                new_pos = (new_pos + 1) % len(self.items)
+
+                # Break if we loop back to the original position
+                if new_pos == original_pos:
+                    break
+
+            self.cursor_pos = max(0, min(new_pos, len(self.items)-1))
+
+        # Initialise sheets
+        extend_list_to_length(self.sheet_states, length=len(self.sheets), default_value={})
 
-        # Sheets and files
         if len(self.sheet_states) < len(self.sheets):
             self.sheet_states += [{} for _ in range(len(self.sheets) - len(self.sheet_states))]
         if len(self.sheets):
@@ -723,15 +806,16 @@ class Picker:
                 self.sheet_index = 0
             self.sheet_name = self.sheets[self.sheet_index]
 
-        if len(self.loaded_file_states) < len(self.loaded_files):
-            self.loaded_file_states += [{} for _ in range(len(self.loaded_files) - len(self.loaded_file_states))]
+        # Initialise files
+        extend_list_to_length(self.loaded_file_states, length=len(self.loaded_files), default_value={})
         if len(self.loaded_files):
             if self.loaded_file_index >= len(self.loaded_files):
                 self.loaded_file_index = 0
             self.loaded_file = self.loaded_files[self.loaded_file_index]
 
-        # if tracking and len(self.items) > 1:
-        # Ensure that selected indices are tracked upon data refresh
+
+        # Ensure that the correct cursor_pos and selected indices are reselected
+        #   if  we have fetched new data.
         if self.track_entries_upon_refresh and (self.data_ready or tracking) and len(self.items) > 1:
             selected_indices = []
             all_ids = [item[self.id_column] for item in self.items]
@@ -754,6 +838,8 @@ class Picker:
 
 
 
+            # Ensure cursor_pos is set to a valid index
+            # If we have fetched new data then we attempt to set cursor_pos to the row with the same id as prev
             if len(self.indexed_items):
                 if self.pin_cursor:
                     self.cursor_pos = min(self.cursor_pos_prev, len(self.indexed_items)-1)
@@ -765,7 +851,12 @@ class Picker:
             else:
                 self.cursor_pos = 0
 
-        self.right_pane_index = max(0, min(self.right_pane_index, len(self.right_panes)-1))
+
+        # Ensure that the pane indices are within the range of the available panes.
+        if len(self.left_panes): self.left_pane_index %= len(self.left_panes)
+        else: self.left_pane_index = 0
+        if len(self.right_panes): self.right_pane_index %= len(self.right_panes)
+        else: self.right_pane_index = 0
         
 
 
@@ -862,14 +953,11 @@ class Picker:
             self.stdscr.erase()
 
         self.update_term_size()
-        # if self.split_right and len(self.right_panes):
-        #     proportion = self.right_panes[self.right_pane_index]["proportion"]
-        #     self.rows_w, self.rows_h = int(self.term_w*proportion), self.term_h
-        # else:
-        #     self.rows_w, self.rows_h = self.term_w, self.term_h
 
-        # The height of the footer may need to be adjusted if the file changes.
+        # Determine footer size
         self.footer.adjust_sizes(self.term_h,self.term_w)
+
+        # The height of the footer may need to be adjusted if the file changes.
         self.calculate_section_sizes()
 
         # Test if the terminal is of a sufficient size to display the picker
@@ -887,28 +975,18 @@ class Picker:
             end_index = min(start_index + self.items_per_page, len(self.indexed_items))
         if len(self.indexed_items) == 0: start_index, end_index = 0, 0
 
-        # self.column_widths = get_column_widths(self.items, header=self.header, max_column_width=self.max_column_width, number_columns=self.number_columns, max_total_width=w)
-        # Determine widths based only on the currently indexed rows
-        # rows = [v[1] for v in self.indexed_items] if len(self.indexed_items) else self.items
-        # Determine widths based only on the currently displayed indexed rows
-        # rows = [v[1] for v in self.indexed_items[start_index:end_index]] if len(self.indexed_items) else self.items
         self.get_visible_rows()
         self.column_widths = get_column_widths(self.visible_rows, header=self.header, max_column_width=self.max_column_width, number_columns=self.number_columns, max_total_width=self.rows_w, unicode_char_width=self.unicode_char_width)
-        visible_column_widths = [c for i,c in enumerate(self.column_widths) if i not in self.hidden_columns]
-        visible_columns_total_width = sum(visible_column_widths) + len(self.separator)*(len(visible_column_widths)-1)
-
-        # Determine the number of items_per_page, top_size and bottom_size
-        # self.calculate_section_sizes()
+        self.visible_column_widths = [c for i,c in enumerate(self.column_widths) if i not in self.hidden_columns]
+        visible_columns_total_width = sum(self.visible_column_widths) + len(self.separator)*(len(self.visible_column_widths)-1)
 
-        # top_space = self.top_gap
 
-        ## Display title (if applicable)
+        ## Display title
         if self.title:
             padded_title = f" {self.title.strip()} "
             self.stdscr.addstr(self.top_gap, 0, f"{' ':^{self.term_w}}", curses.color_pair(self.colours_start+16))
             title_x = (self.term_w-wcswidth(padded_title))//2
             self.stdscr.addstr(self.top_gap, title_x, padded_title, curses.color_pair(self.colours_start+16) | curses.A_BOLD)
-            # top_space += 1
 
         ## Display modes
         if self.display_modes and self.modes not in [[{}], []]:
@@ -930,7 +1008,6 @@ class Picker:
                 else:
                     self.stdscr.addstr(self.top_gap+1, xmode, mode_str, curses.color_pair(self.colours_start+15) | curses.A_UNDERLINE)
                 xmode += split_space+mode_widths[i]
-            # top_space += 1
 
         ## Display header
         if self.header and self.show_header:
@@ -948,93 +1025,75 @@ class Picker:
 
 
                 header_str += f"{col_str:^{self.column_widths[i]-len(number)}}"
-                header_str += self.header_separator
+                if i == self.selected_column-1:
+                    header_str += self.header_separator_before_selected_column
+                else:
+                    header_str += self.header_separator
                 header_str_w = min(self.rows_w-self.left_gutter_width, visible_columns_total_width+1, self.term_w-self.startx)
 
             header_str = header_str[self.leftmost_char:]
             header_str = header_str[:header_str_w]
             header_ypos = self.top_gap + bool(self.title) + bool(self.display_modes and self.modes)
+
+            # Ensure that the full header width is filled--important if the header rows do not fill the terminal width
             self.stdscr.addstr(header_ypos, self.rows_box_x_i, ' '*self.rows_w, curses.color_pair(self.colours_start+28) | curses.A_BOLD)
+
+            # Draw header string
             self.stdscr.addstr(header_ypos, self.startx, header_str, curses.color_pair(self.colours_start+4) | curses.A_BOLD)
 
             # Highlight sort column
-            try:
-                if self.selected_column != None and self.selected_column not in self.hidden_columns:
-                    # start of string is on screen
-                    col_width = self.column_widths[self.selected_column]
-                    number = f"{self.selected_column}. " if self.number_columns else ""
-                    col_str = self.header[self.selected_column][:self.column_widths[self.selected_column]-len(number)]
-                    highlighted_col_str = (number+f"{col_str:^{self.column_widths[self.selected_column]-len(number)}}") + self.separator
-
-                    if len(self.column_widths) == 1:
-                        colour = curses.color_pair(self.colours_start+28) | curses.A_BOLD
-                    else:
-                        colour = curses.color_pair(self.colours_start+19) | curses.A_BOLD
-                    # Start of selected column is on the screen
-                    if self.leftmost_char <= len(up_to_selected_col) and self.leftmost_char+self.rows_w-self.left_gutter_width > len(up_to_selected_col):
-                        x_pos = len(up_to_selected_col) - self.leftmost_char + self.startx
+            if self.selected_column != None and self.selected_column not in self.hidden_columns:
+                # start of string is on screen
+                col_width = self.column_widths[self.selected_column]
+                number = f"{self.selected_column}. " if self.number_columns else ""
+                col_str = self.header[self.selected_column][:self.column_widths[self.selected_column]-len(number)]
+                highlighted_col_str = (number+f"{col_str:^{self.column_widths[self.selected_column]-len(number)}}") + self.separator
+
+                if len(self.column_widths) == 1:
+                    colour = curses.color_pair(self.colours_start+28) | curses.A_BOLD
+                else:
+                    colour = curses.color_pair(self.colours_start+19) | curses.A_BOLD
+                # Start of selected column is on the screen
+                if self.leftmost_char <= len(up_to_selected_col) and self.leftmost_char+self.rows_w-self.left_gutter_width > len(up_to_selected_col):
+                    x_pos = len(up_to_selected_col) - self.leftmost_char + self.startx
 
-                        # Whole cell of the selected column is on the screen
-                        if len(up_to_selected_col)+col_width - self.leftmost_char < self.rows_w-self.left_gutter_width:
-                            disp_str = highlighted_col_str
+                    # Whole cell of the selected column is on the screen
+                    if len(up_to_selected_col)+col_width - self.leftmost_char < self.rows_w-self.left_gutter_width:
+                        disp_str = highlighted_col_str
 
-                        # Start of the cell is on the screen, but the end of the cell is not
-                        else:
-                            overflow = (len(up_to_selected_col)+len(highlighted_col_str)) - (self.leftmost_char+self.rows_w - self.left_gutter_width)
-                            disp_str = highlighted_col_str[:-overflow]
-                        disp_str_w = min(len(disp_str), self.term_w-x_pos)
-                        disp_str = truncate_to_display_width(disp_str, disp_str_w, self.centre_in_cols, self.unicode_char_width)
-
-                        self.stdscr.addstr(header_ypos, x_pos , disp_str, colour)
-                    # Start of the cell is to the right of the screen
-                    elif self.leftmost_char+self.rows_w <= len(up_to_selected_col):
-                        pass
-                    # The end of the cell is on the screen, the start of the cell is not
-                    elif 0 <= len(up_to_selected_col)+col_width - self.leftmost_char <= self.rows_w :
-                        x_pos = self.startx
-                        beg = self.leftmost_char - len(up_to_selected_col)
-                        disp_str = highlighted_col_str[beg:]
-                        disp_str_w = min(len(disp_str), self.term_w-x_pos)
-                        disp_str = truncate_to_display_width(disp_str, disp_str_w, self.centre_in_cols, self.unicode_char_width)
-                        self.stdscr.addstr(header_ypos, x_pos , disp_str, colour)
-                    # The middle of the cell is on the screen, the start and end of the cell are not
-                    elif self.leftmost_char <= len(up_to_selected_col) + col_width//2 <= self.leftmost_char+self.rows_w:
-                        beg = self.leftmost_char - len(up_to_selected_col)
-                        overflow = (len(up_to_selected_col)+len(highlighted_col_str)) - (self.leftmost_char+self.rows_w)
-                        x_pos = self.startx
-                        disp_str = highlighted_col_str[beg:-overflow]
-                        disp_str_w = min(len(disp_str), self.term_w-x_pos)
-                        disp_str = truncate_to_display_width(disp_str, disp_str_w, self.centre_in_cols, self.unicode_char_width)
-
-                        self.stdscr.addstr(header_ypos, x_pos , disp_str, colour)
-                    # The cell is to the left of the screen
+                    # Start of the cell is on the screen, but the end of the cell is not
                     else:
-                        pass
-
-                    # elif self.leftmost_char:
-                    #     os.system(f"notify-send 'cell is to the right of the screen'")
-
-                    #
-                    #
-                    # if len(self.header) > 1 and (len(up_to_selected_col)-self.leftmost_char) < self.rows_w:
-                    #     number = f"{self.selected_column}. " if self.number_columns else ""
-                    #     # number = f"{intStringToExponentString(self.selected_column)}. " if self.number_columns else ""
-                    #     # self.startx + len(up_to_selected_col) - self.leftmost_char
-                    #     highlighed_col_startx = max(self.startx, self.startx + len(up_to_selected_col) - self.leftmost_char)
-                    #
-                    #
-                    #     col_str = self.header[self.selected_column][:self.column_widths[self.selected_column]-len(number)]
-                    #     highlighted_col_str = (number+f"{col_str:^{self.column_widths[self.selected_column]}}") + self.separator
-                    #     end_of_highlighted_col_str = self.rows_w-(highlighed_col_startx+len(highlighted_col_str)) if (highlighed_col_startx+len(highlighted_col_str)) > self.rows_w else len(highlighted_col_str)
-                    #     if (highlighed_col_startx+len(highlighted_col_str)) > self.rows_w:
-                    #         end_of_highlighted_col_str = self.rows_w-(highlighed_col_startx+len(highlighted_col_str)) 
-                    #     else:
-                    #         end_of_highlighted_col_str = len(highlighted_col_str)
-                    #
-                    #     start_of_highlighted_col_str = max(self.leftmost_char - len(up_to_selected_col), 0)
-                    #     self.stdscr.addstr(header_ypos, highlighed_col_startx , highlighted_col_str[start_of_highlighted_col_str:end_of_highlighted_col_str][:self.column_widths[self.selected_column]+len(self.separator)], curses.color_pair(self.colours_start+19) | curses.A_BOLD)
-            except:
-                pass
+                        overflow = (len(up_to_selected_col)+len(highlighted_col_str)) - (self.leftmost_char+self.rows_w - self.left_gutter_width)
+                        disp_str = highlighted_col_str[:-overflow]
+                    disp_str_w = min(len(disp_str), self.term_w-x_pos)
+                    disp_str = truncate_to_display_width(disp_str, disp_str_w, self.centre_in_cols, self.unicode_char_width)
+
+                    self.stdscr.addstr(header_ypos, x_pos , disp_str, colour)
+                # Start of the cell is to the right of the screen
+                elif self.leftmost_char+self.rows_w <= len(up_to_selected_col):
+                    pass
+                # The end of the cell is on the screen, the start of the cell is not
+                elif 0 <= len(up_to_selected_col)+col_width - self.leftmost_char <= self.rows_w :
+                    x_pos = self.startx
+                    beg = self.leftmost_char - len(up_to_selected_col)
+                    disp_str = highlighted_col_str[beg:]
+                    disp_str_w = min(len(disp_str), self.term_w-x_pos)
+                    disp_str = truncate_to_display_width(disp_str, disp_str_w, self.centre_in_cols, self.unicode_char_width)
+                    self.stdscr.addstr(header_ypos, x_pos , disp_str, colour)
+                # The middle of the cell is on the screen, the start and end of the cell are not
+                elif self.leftmost_char <= len(up_to_selected_col) + col_width//2 <= self.leftmost_char+self.rows_w:
+                    beg = self.leftmost_char - len(up_to_selected_col)
+                    overflow = (len(up_to_selected_col)+len(highlighted_col_str)) - (self.leftmost_char+self.rows_w)
+                    x_pos = self.startx
+                    disp_str = highlighted_col_str[beg:-overflow]
+                    disp_str_w = min(len(disp_str), self.term_w-x_pos)
+                    disp_str = truncate_to_display_width(disp_str, disp_str_w, self.centre_in_cols, self.unicode_char_width)
+
+                    self.stdscr.addstr(header_ypos, x_pos , disp_str, colour)
+
+                # The cell is to the left of the focused part of the screen
+                else:
+                    pass
 
         # Display row header 
         if self.show_row_header:
@@ -1058,41 +1117,34 @@ class Picker:
                 colour = curses.color_pair(self.colours_start+colour_pair_number) | curses.A_BOLD
             else:
                 colour = curses.color_pair(self.colours_start+colour_pair_number)
-            try:
-                # Start of cell is on screen
-                if self.startx <= cell_pos <= self.rows_w+self.startx:
-                    s = "max" if cell_max_width <= cell_width else "norm"
-                    self.stdscr.addstr(y, cell_pos, (' '*cell_width)[:cell_max_width], colour)
-                    if self.centre_in_cols:
-                        cell_value = f"{self.indexed_items[row][1][col]:^{cell_width-len(self.separator)}}" + self.separator
-                    else:
-                        cell_value = self.indexed_items[row][1][col][:self.column_widths[col]] + self.separator
-                    # cell_value = cell_value[:min(cell_width, cell_max_width)-len(self.separator)]
-                    cell_value = truncate_to_display_width(cell_value, min(cell_width, cell_max_width), self.centre_in_cols, self.unicode_char_width)
-                    # cell_value = cell_value + self.separator
-                    # cell_value = cell_value
-                    cell_value = truncate_to_display_width(cell_value, min(cell_width, cell_max_width), self.centre_in_cols, self.unicode_char_width)
-                    if wcswidth(cell_value) + cell_pos > self.term_w:
-                        cell_value = truncate_to_display_width(cell_value, self.term_w-cell_pos-10, self.centre_in_cols, self.unicode_char_width)
-
-                    self.stdscr.addstr(y, cell_pos, cell_value, colour)
-                # Part of the cell is on screen
-                elif self.startx <= cell_pos+cell_width and cell_pos <= (self.rows_w):
-                    s = "max" if cell_max_width <= cell_width else "norm"
-                    cell_start = self.startx - cell_pos
-                    # self.stdscr.addstr(y, self.startx, ' '*(cell_width-cell_start), curses.color_pair(self.colours_start+colour_pair_number))
-                    cell_value = self.indexed_items[row][1][col]
-                    cell_value = f"{cell_value:^{self.column_widths[col]}}"
-
-                    cell_value = cell_value[cell_start:visible_column_widths[col]][:self.rows_w-self.left_gutter_width]
-                    cell_value = truncate_to_display_width(cell_value, min(wcswidth(cell_value), cell_width, cell_max_width), self.centre_in_cols, self.unicode_char_width)
-                    cell_value += self.separator
-                    cell_value = truncate_to_display_width(cell_value, min(wcswidth(cell_value), cell_width, cell_max_width), self.centre_in_cols, self.unicode_char_width)
-                    self.stdscr.addstr(y, self.startx, cell_value, colour)
+            # Start of cell is on screen
+            if self.startx <= cell_pos <= self.rows_w+self.startx:
+                s = "max" if cell_max_width <= cell_width else "norm"
+                self.stdscr.addstr(y, cell_pos, (' '*cell_width)[:cell_max_width], colour)
+                if self.centre_in_cols:
+                    cell_value = f"{self.indexed_items[row][1][col]:^{cell_width-len(self.separator)}}" + self.separator
                 else:
-                    pass
-                # if colour_pair_number == 5:
-            except:
+                    cell_value = self.indexed_items[row][1][col][:self.column_widths[col]] + self.separator
+                cell_value = truncate_to_display_width(cell_value, min(cell_width, cell_max_width), self.centre_in_cols, self.unicode_char_width)
+                cell_value = truncate_to_display_width(cell_value, min(cell_width, cell_max_width), self.centre_in_cols, self.unicode_char_width)
+                if wcswidth(cell_value) + cell_pos > self.term_w:
+                    cell_value = truncate_to_display_width(cell_value, self.term_w-cell_pos-10, self.centre_in_cols, self.unicode_char_width)
+
+                self.stdscr.addstr(y, cell_pos, cell_value, colour)
+
+            # Part of the cell is on screen
+            elif self.startx <= cell_pos+cell_width and cell_pos <= (self.rows_w):
+                s = "max" if cell_max_width <= cell_width else "norm"
+                cell_start = self.startx - cell_pos
+                cell_value = self.indexed_items[row][1][col]
+                cell_value = f"{cell_value:^{self.column_widths[col]}}"
+
+                cell_value = cell_value[cell_start:visible_column_widths[col]][:self.rows_w-self.left_gutter_width]
+                cell_value = truncate_to_display_width(cell_value, min(wcswidth(cell_value), cell_width, cell_max_width), self.centre_in_cols, self.unicode_char_width)
+                cell_value += self.separator
+                cell_value = truncate_to_display_width(cell_value, min(wcswidth(cell_value), cell_width, cell_max_width), self.centre_in_cols, self.unicode_char_width)
+                self.stdscr.addstr(y, self.startx, cell_value, colour)
+            else:
                 pass
 
 
@@ -1136,10 +1188,11 @@ class Picker:
                         match = re.search(highlight["match"], truncate_to_display_width(item[1][highlight["field"]], self.column_widths[highlight["field"]], centre=False, unicode_char_width=self.unicode_char_width), re.IGNORECASE)
                         if not match: continue
                         field_start = sum([width for i, width in enumerate(self.column_widths[:highlight["field"]]) if i not in self.hidden_columns]) + sum([1 for i in range(highlight["field"]) if i not in self.hidden_columns])*wcswidth(self.separator)
+                        width = min(self.column_widths[highlight["field"]]-(field_start-self.leftmost_char), self.rows_w-self.left_gutter_width)
 
                         ## We want to search the non-centred values but highlight the centred values.
                         if self.centre_in_cols:
-                            tmp = truncate_to_display_width(item[1][highlight["field"]], self.column_widths[highlight["field"]], self.centre_in_cols, self.unicode_char_width)
+                            tmp = truncate_to_display_width(item[1][highlight["field"]], width, self.centre_in_cols, self.unicode_char_width)
                             field_start += (len(tmp) - len(tmp.lstrip()))
 
                         highlight_start = field_start + match.start()
@@ -1150,7 +1203,7 @@ class Picker:
                         continue
                     highlight_start -= self.leftmost_char
                     highlight_end -= self.leftmost_char
-                    self.stdscr.addstr(y, max(self.startx, self.startx+highlight_start), row_str[max(highlight_start,0):min(self.rows_w-self.startx, highlight_end)], curses.color_pair(self.colours_start+highlight["color"]) | curses.A_BOLD)
+                    self.stdscr.addstr(y, max(self.startx, self.startx+highlight_start), row_str[max(highlight_start,0):min(self.rows_w-self.left_gutter_width, highlight_end)], curses.color_pair(self.colours_start+highlight["color"]) | curses.A_BOLD)
                 except:
                     pass
 
@@ -1166,7 +1219,7 @@ class Picker:
         l0_highlights, l1_highlights, l2_highlights = sort_highlights(self.highlights)
 
 
-        row_width = sum(self.column_widths) + len(self.separator)*(len(self.column_widths)-1)
+        row_width = sum(self.visible_column_widths) + len(self.separator)*(len(self.visible_column_widths)-1)
         for idx in range(start_index, end_index):
             item = self.indexed_items[idx]
             y = idx - start_index + self.top_space
@@ -1186,7 +1239,7 @@ class Picker:
             #     trunc_width = 0
 
 
-            trunc_width = min(self.rows_w-self.left_gutter_width, row_width, self.term_w - self.startx)
+            trunc_width = max(0, min(self.rows_w-self.left_gutter_width, row_width, self.term_w - self.startx))
 
             row_str = truncate_to_display_width(row_str_left_adj, trunc_width, self.unicode_char_width)
             # row_str = truncate_to_display_width(row_str, min(w-self.startx, visible_columns_total_width))[self.leftmost_char:]
@@ -1197,7 +1250,7 @@ class Picker:
 
             ## Highlight column
             if self.crosshair_cursor:
-                highlight_cell(idx, self.selected_column, visible_column_widths, colour_pair_number=27, bold=False, y=y)
+                highlight_cell(idx, self.selected_column, self.visible_column_widths, colour_pair_number=27, bold=False, y=y)
                 if idx == self.cursor_pos:
                     self.stdscr.addstr(y, self.startx, row_str[:min(self.rows_w-self.startx, visible_columns_total_width)], curses.color_pair(self.colours_start+27))
 
@@ -1211,21 +1264,21 @@ class Picker:
                 # self.selected_cells_by_row = get_selected_cells_by_row(self.cell_selections)
                 if item[0] in self.selected_cells_by_row:
                     for j in self.selected_cells_by_row[item[0]]:
-                        highlight_cell(idx, j, visible_column_widths, colour_pair_number=25, bold=False, y=y)
+                        highlight_cell(idx, j, self.visible_column_widths, colour_pair_number=25, bold=False, y=y)
 
                 # Visually selected
                 if self.is_selecting:
                     if self.start_selection <= idx <= self.cursor_pos or self.start_selection >= idx >= self.cursor_pos:
                         x_interval = range(min(self.start_selection_col, self.selected_column), max(self.start_selection_col, self.selected_column)+1)
                         for col in x_interval:
-                            highlight_cell(idx, col, visible_column_widths, colour_pair_number=25, bold=False, y=y)
+                            highlight_cell(idx, col, self.visible_column_widths, colour_pair_number=25, bold=False, y=y)
 
                 # Visually deslected
                 if self.is_deselecting:
                     if self.start_selection >= idx >= self.cursor_pos or self.start_selection <= idx <= self.cursor_pos:
                         x_interval = range(min(self.start_selection_col, self.selected_column), max(self.start_selection_col, self.selected_column)+1)
                         for col in x_interval:
-                            highlight_cell(idx, col, visible_column_widths, colour_pair_number=26, bold=False,y=y)
+                            highlight_cell(idx, col, self.visible_column_widths, colour_pair_number=26, bold=False,y=y)
             # Higlight cursor row and selected rows
             elif self.highlight_full_row:
                 if self.selections[item[0]]:
@@ -1242,16 +1295,30 @@ class Picker:
 
             # Highlight the cursor row and the first char of the selected rows.
             else:
-                if self.selections[item[0]]:
-                    self.stdscr.addstr(y, max(self.startx-2,0), ' ', curses.color_pair(self.colours_start+1))
-                # Visually selected
-                if self.is_selecting:
-                    if self.start_selection <= idx <= self.cursor_pos or self.start_selection >= idx >= self.cursor_pos:
+                if self.selected_char:
+                    if self.selections[item[0]]:
+                        self.stdscr.addstr(y, max(self.startx-2,0), self.selected_char, curses.color_pair(self.colours_start+2))
+                    else:
+                        self.stdscr.addstr(y, max(self.startx-2,0), self.unselected_char, curses.color_pair(self.colours_start+2))
+                    # Visually selected
+                    if self.is_selecting:
+                        if self.start_selection <= idx <= self.cursor_pos or self.start_selection >= idx >= self.cursor_pos:
+                            self.stdscr.addstr(y, max(self.startx-2,0), self.selecting_char, curses.color_pair(self.colours_start+2))
+                    # Visually deslected
+                    if self.is_deselecting:
+                        if self.start_selection >= idx >= self.cursor_pos or self.start_selection <= idx <= self.cursor_pos:
+                            self.stdscr.addstr(y, max(self.startx-2,0), self.deselecting_char, curses.color_pair(self.colours_start+2))
+                else:
+                    if self.selections[item[0]]:
                         self.stdscr.addstr(y, max(self.startx-2,0), ' ', curses.color_pair(self.colours_start+1))
-                # Visually deslected
-                if self.is_deselecting:
-                    if self.start_selection >= idx >= self.cursor_pos or self.start_selection <= idx <= self.cursor_pos:
-                        self.stdscr.addstr(y, max(self.startx-2,0), ' ', curses.color_pair(self.colours_start+10))
+                    # Visually selected
+                    if self.is_selecting:
+                        if self.start_selection <= idx <= self.cursor_pos or self.start_selection >= idx >= self.cursor_pos:
+                            self.stdscr.addstr(y, max(self.startx-2,0), ' ', curses.color_pair(self.colours_start+1))
+                    # Visually deslected
+                    if self.is_deselecting:
+                        if self.start_selection >= idx >= self.cursor_pos or self.start_selection <= idx <= self.cursor_pos:
+                            self.stdscr.addstr(y, max(self.startx-2,0), ' ', curses.color_pair(self.colours_start+10))
 
             if not self.highlights_hide:
                 draw_highlights(l1_highlights, idx, y, item)
@@ -1261,7 +1328,7 @@ class Picker:
             # Draw cursor
             if idx == self.cursor_pos:
                 if self.cell_cursor:
-                    highlight_cell(idx, self.selected_column, visible_column_widths, colour_pair_number=5, bold=True, y=y)
+                    highlight_cell(idx, self.selected_column, self.visible_column_widths, colour_pair_number=5, bold=True, y=y)
                 else:
                     self.stdscr.addstr(y, self.startx, row_str[:self.rows_w-self.left_gutter_width], curses.color_pair(self.colours_start+5) | curses.A_BOLD)
 
@@ -1366,6 +1433,21 @@ class Picker:
             # self.stdscr.timeout(2000)  # timeout is set to 50 in order to get the infobox to be displayed so here we reset it to 2000
 
 
+    def refresh_and_draw_screen(self):
+        """
+        Clears and refreshes the screen, restricts and unrestricts curses, 
+            ensures correct terminal settings, and then draws the screen.
+        """
+
+        self.logger.info(f"key_function redraw_screen")
+        self.stdscr.clear()
+        self.stdscr.refresh()
+        restrict_curses(self.stdscr)
+        unrestrict_curses(self.stdscr)
+        self.stdscr.clear()
+        self.stdscr.refresh()
+
+        self.draw_screen()
 
     def infobox(self, stdscr: curses.window, message: str ="", title: str ="Infobox",  colours_end: int = 0, duration: int = 4) -> curses.window:
         """ Display non-interactive infobox window. """
@@ -1421,116 +1503,126 @@ class Picker:
         self.logger.debug(f"function: get_function_data()")
         """ Returns a dict of the main variables needed to restore the state of list_pikcer. """
         function_data = {
-            "selections":                       self.selections,
-            "cell_selections":                  self.cell_selections,
-            "selected_cells_by_row":            self.selected_cells_by_row,
-            "items_per_page":                   self.items_per_page,
-            "current_row":                      self.current_row,
-            "current_page":                     self.current_page,
-            "cursor_pos":                       self.cursor_pos,
-            "colours":                          self.colours,
-            "colour_theme_number":              self.colour_theme_number,
-            "selected_column":                  self.selected_column,
-            "sort_column":                      self.sort_column,
-            "sort_method":                      self.sort_method,
-            "sort_reverse":                     self.sort_reverse,
-            "SORT_METHODS":                     self.SORT_METHODS,
-            "hidden_columns":                   self.hidden_columns,
-            "is_selecting":                     self.is_selecting,
-            "is_deselecting":                   self.is_deselecting,
-            "user_opts":                        self.user_opts,
-            "options_list":                     self.options_list,
-            "user_settings":                    self.user_settings,
-            "separator":                        self.separator,
-            "search_query":                     self.search_query,
-            "search_count":                     self.search_count,
-            "search_index":                     self.search_index,
-            "filter_query":                     self.filter_query,
-            "indexed_items":                    self.indexed_items,
-            "start_selection":                  self.start_selection,
-            "start_selection_col":              self.start_selection_col,
-            "end_selection":                    self.end_selection,
-            "highlights":                       self.highlights,
-            "max_column_width":                 self.max_column_width,
-            "column_indices":                   self.column_indices,
-            "mode_index":                       self.mode_index,
-            "modes":                            self.modes,
-            "title":                            self.title,
-            "display_modes":                    self.display_modes,
-            "require_option":                   self.require_option,
-            "require_option_default":           self.require_option_default,
-            "option_functions":                 self.option_functions,
-            "top_gap":                          self.top_gap,
-            "number_columns":                   self.number_columns,
-            "items":                            self.items,
-            "indexed_items":                    self.indexed_items,
-            "header":                           self.header,
-            "scroll_bar":                       self.scroll_bar,
-            "columns_sort_method":              self.columns_sort_method,
-            "disabled_keys":                    self.disabled_keys,
-            "show_footer":                      self.show_footer,
-            "footer_string":                    self.footer_string,
-            "footer_string_auto_refresh":       self.footer_string_auto_refresh,
-            "footer_string_refresh_function":   self.footer_string_refresh_function,
-            "footer_timer":                     self.footer_timer,
-            "footer_style":                     self.footer_style,
-            "colours_start":                    self.colours_start,
-            "colours_end":                      self.colours_end,
-            "display_only":                     self.display_only,
-            "infobox_items":                    self.infobox_items,
-            "display_infobox":                  self.display_infobox,
-            "infobox_title":                    self.infobox_title,
-            "key_remappings":                   self.key_remappings,
-            "auto_refresh":                     self.auto_refresh,
-            "get_new_data":                     self.get_new_data,
-            "refresh_function":                 self.refresh_function,
-            "timer":                            self.timer,
-            "get_data_startup":                 self.get_data_startup,
-            "get_footer_string_startup":        self.get_footer_string_startup,
-            "editable_columns":                 self.editable_columns,
-            "last_key":                         self.last_key,
-            "centre_in_terminal":               self.centre_in_terminal,
-            "centre_in_terminal_vertical":      self.centre_in_terminal_vertical,
-            "centre_in_cols":                   self.centre_in_cols,
-            "highlight_full_row":               self.highlight_full_row,
-            "cell_cursor":                      self.cell_cursor,
-            "column_widths":                    self.column_widths,
-            "track_entries_upon_refresh":       self.track_entries_upon_refresh,
-            "pin_cursor":                       self.pin_cursor,
-            "id_column":                        self.id_column,
-            "startup_notification":             self.startup_notification,
-            "keys_dict":                        self.keys_dict,
-            "cancel_is_back":                   self.cancel_is_back,
-            "paginate":                         self.paginate,
-            "leftmost_char":                    self.leftmost_char,
-            "history_filter_and_search" :       self.history_filter_and_search,
-            "history_pipes" :                   self.history_pipes,
-            "history_opts" :                    self.history_opts,
-            "history_edits" :                   self.history_edits,
-            "history_settings":                 self.history_settings,
-            "show_header":                      self.show_header,
-            "show_row_header":                  self.show_row_header,
-            "debug":                            self.debug,
-            "debug_level":                      self.debug_level,
-            "reset_colours":                    self.reset_colours,
-            "unicode_char_width":               self.unicode_char_width,
-            "command_stack":                    self.command_stack,
-            "loaded_file":                      self.loaded_file,
-            "loaded_files":                     self.loaded_files,
-            "loaded_file_index":                self.loaded_file_index,
-            "loaded_file_states":               self.loaded_file_states,
-            "sheet_index":                      self.sheet_index,
-            "sheets":                           self.sheets,
-            "sheet_name":                       self.sheet_name,
-            "sheet_states":                     self.sheet_states,
-            "split_right":                      self.split_right,
-            "right_panes":                      self.right_panes,
-            "right_pane_index":                 self.right_pane_index,
-            "split_left":                       self.split_left,
-            "left_panes":                       self.left_panes,
-            "left_pane_index":                  self.left_pane_index,
-            "crosshair_cursor":                 self.crosshair_cursor,
-
+            "self":                                     self,
+            "selections":                               self.selections,
+            "cell_selections":                          self.cell_selections,
+            "selected_cells_by_row":                    self.selected_cells_by_row,
+            "items_per_page":                           self.items_per_page,
+            "current_row":                              self.current_row,
+            "current_page":                             self.current_page,
+            "cursor_pos":                               self.cursor_pos,
+            "colours":                                  self.colours,
+            "colour_theme_number":                      self.colour_theme_number,
+            "selected_column":                          self.selected_column,
+            "sort_column":                              self.sort_column,
+            "sort_method":                              self.sort_method,
+            "sort_reverse":                             self.sort_reverse,
+            "SORT_METHODS":                             self.SORT_METHODS,
+            "hidden_columns":                           self.hidden_columns,
+            "is_selecting":                             self.is_selecting,
+            "is_deselecting":                           self.is_deselecting,
+            "user_opts":                                self.user_opts,
+            "options_list":                             self.options_list,
+            "user_settings":                            self.user_settings,
+            "separator":                                self.separator,
+            "header_separator":                         self.header_separator,
+            "header_separator_before_selected_column":  self.header_separator_before_selected_column,
+            "search_query":                             self.search_query,
+            "search_count":                             self.search_count,
+            "search_index":                             self.search_index,
+            "filter_query":                             self.filter_query,
+            "indexed_items":                            self.indexed_items,
+            "start_selection":                          self.start_selection,
+            "start_selection_col":                      self.start_selection_col,
+            "end_selection":                            self.end_selection,
+            "highlights":                               self.highlights,
+            "max_column_width":                         self.max_column_width,
+            "column_indices":                           self.column_indices,
+            "mode_index":                               self.mode_index,
+            "modes":                                    self.modes,
+            "title":                                    self.title,
+            "display_modes":                            self.display_modes,
+            "require_option":                           self.require_option,
+            "require_option_default":                   self.require_option_default,
+            "option_functions":                         self.option_functions,
+            "top_gap":                                  self.top_gap,
+            "number_columns":                           self.number_columns,
+            "items":                                    self.items,
+            "indexed_items":                            self.indexed_items,
+            "header":                                   self.header,
+            "scroll_bar":                               self.scroll_bar,
+            "columns_sort_method":                      self.columns_sort_method,
+            "disabled_keys":                            self.disabled_keys,
+            "show_footer":                              self.show_footer,
+            "footer_string":                            self.footer_string,
+            "footer_string_auto_refresh":               self.footer_string_auto_refresh,
+            "footer_string_refresh_function":           self.footer_string_refresh_function,
+            "footer_timer":                             self.footer_timer,
+            "footer_style":                             self.footer_style,
+            "colours_start":                            self.colours_start,
+            "colours_end":                              self.colours_end,
+            "display_only":                             self.display_only,
+            "infobox_items":                            self.infobox_items,
+            "display_infobox":                          self.display_infobox,
+            "infobox_title":                            self.infobox_title,
+            "key_remappings":                           self.key_remappings,
+            "auto_refresh":                             self.auto_refresh,
+            "get_new_data":                             self.get_new_data,
+            "refresh_function":                         self.refresh_function,
+            "timer":                                    self.timer,
+            "get_data_startup":                         self.get_data_startup,
+            "get_footer_string_startup":                self.get_footer_string_startup,
+            "editable_columns":                         self.editable_columns,
+            "last_key":                                 self.last_key,
+            "centre_in_terminal":                       self.centre_in_terminal,
+            "centre_in_terminal_vertical":              self.centre_in_terminal_vertical,
+            "centre_in_cols":                           self.centre_in_cols,
+            "highlight_full_row":                       self.highlight_full_row,
+            "cell_cursor":                              self.cell_cursor,
+            "column_widths":                            self.column_widths,
+            "track_entries_upon_refresh":               self.track_entries_upon_refresh,
+            "pin_cursor":                               self.pin_cursor,
+            "id_column":                                self.id_column,
+            "startup_notification":                     self.startup_notification,
+            "keys_dict":                                self.keys_dict,
+            "cancel_is_back":                           self.cancel_is_back,
+            "paginate":                                 self.paginate,
+            "leftmost_char":                            self.leftmost_char,
+            "history_filter_and_search" :               self.history_filter_and_search,
+            "history_pipes" :                           self.history_pipes,
+            "history_opts" :                            self.history_opts,
+            "history_edits" :                           self.history_edits,
+            "history_settings":                         self.history_settings,
+            "show_header":                              self.show_header,
+            "show_row_header":                          self.show_row_header,
+            "debug":                                    self.debug,
+            "debug_level":                              self.debug_level,
+            "reset_colours":                            self.reset_colours,
+            "unicode_char_width":                       self.unicode_char_width,
+            "command_stack":                            self.command_stack,
+            "loaded_file":                              self.loaded_file,
+            "loaded_files":                             self.loaded_files,
+            "loaded_file_index":                        self.loaded_file_index,
+            "loaded_file_states":                       self.loaded_file_states,
+            "sheet_index":                              self.sheet_index,
+            "sheets":                                   self.sheets,
+            "sheet_name":                               self.sheet_name,
+            "sheet_states":                             self.sheet_states,
+            "split_right":                              self.split_right,
+            "right_panes":                              self.right_panes,
+            "right_pane_index":                         self.right_pane_index,
+            "split_left":                               self.split_left,
+            "left_panes":                               self.left_panes,
+            "left_pane_index":                          self.left_pane_index,
+            "crosshair_cursor":                         self.crosshair_cursor,
+            "generate_data_for_hidden_columns":         self.generate_data_for_hidden_columns,
+            "thread_stop_event":                        self.thread_stop_event,
+            "data_generation_queue":                    self.data_generation_queue,
+            "process_manager":                          self.process_manager,
+            "threads":                                  self.threads,
+            "processes":                                self.processes,
+            "items_sync_loop_event":                    self.items_sync_loop_event,
+            "items_sync_thread":                        self.items_sync_thread,
         }
         return function_data
 
@@ -1583,15 +1675,6 @@ class Picker:
         self.initialise_picker_state(reset_colours=reset_colours)
 
         self.initialise_variables()
-        # if "colour_theme_number" in function_data:
-        #     global COLOURS_SET
-        #     COLOURS_SET = False
-        #     colours_end = set_colours(pick=self.colour_theme_number, start=self.colours_start)
-                
-        # if "items" in function_data: self.items = function_data["items"]
-        # if "header" in function_data: self.header = function_data["header"]
-        # self.indexed_items = function_data["indexed_items"] if "indexed_items" in function_data else []
-
 
 
     def delete_entries(self) -> None:
@@ -1665,6 +1748,7 @@ class Picker:
             "split_left": False,
             "cell_cursor": False,
             "crosshair_cursor": False,
+            "header_separator": " │",
         }
         while True:
             self.update_term_size()
@@ -1687,6 +1771,102 @@ class Picker:
             return {}, "", f
             
 
+    def select_columns(
+            self,
+            stdscr: curses.window,
+            # options: list[list[str]] =[],
+            # title: str = "Choose option",
+            # x:int=0,
+            # y:int=0,
+            # literal:bool=False,
+            # colours_start:int=0,
+            # header: list[str] = [],
+            # require_option:list = [],
+            # option_functions: list = [],
+    ) -> Tuple[dict, str, dict]:
+        """
+        Display input field at x,y
+
+        ---Arguments
+            stdscr: curses screen
+            usrtxt (str): text to be edited by the user
+            title (str): The text to be displayed at the start of the text option picker
+            x (int): prompt begins at (x,y) in the screen given
+            y (int): prompt begins at (x,y) in the screen given
+            colours_start (bool): start index of curses init_pair.
+
+        ---Returns
+            usrtxt, return_code
+            usrtxt: the text inputted by the user
+            return_code: 
+                            0: user hit escape
+                            1: user hit return
+        """
+        self.logger.info(f"function: select_columns()")
+
+        cursor = 0
+
+        if self.header:
+            columns = [s for i, s in enumerate(self.header)]
+        else:
+            columns = [f"" for i in range(len(self.column_widths))]
+        
+        ## Column info variable
+        columns_set = [[f"{i}", columns[i]] for i in range(len(self.column_widths))]
+        header = ["#", "Column Name"]
+
+        selected = [False if i in self.hidden_columns else True for i in range(len(self.column_widths))]
+        selected = {i: False if i in self.hidden_columns else True for i in range(len(self.column_widths))}
+
+        option_picker_data = {
+            "items": columns_set,
+            "colours": notification_colours,
+            "colours_start": self.notification_colours_start,
+            "title":"Select Columns",
+            "header": header,
+            "hidden_columns":[],
+            # "require_option":require_option,
+            # "keys_dict": options_keys,
+            "selections": selected,
+            "show_footer": False,
+            "cancel_is_back": True,
+            "number_columns": False,
+            "reset_colours": False,
+            "split_right": False,
+            "split_left": False,
+            "cell_cursor": False,
+            "crosshair_cursor": False,
+            "separator": "  ",
+            "header_separator": " │",
+            "header_separator_before_selected_column": " ▐",
+            "selected_char": "☒",
+            "unselected_char": "☐",
+            "selecting_char": "☒",
+            "deselecting_char": "☐",
+        }
+        while True:
+            self.update_term_size()
+
+            choose_opts_widths = get_column_widths(columns_set, unicode_char_width=self.unicode_char_width)
+            window_width = min(max(sum(choose_opts_widths) + 6, 50) + 6, self.term_w)
+            window_height = min(self.term_h//2, max(6, len(columns_set)+3))
+
+            submenu_win = curses.newwin(window_height, window_width, (self.term_h-window_height)//2, (self.term_w-window_width)//2)
+            submenu_win.keypad(True)
+            option_picker_data["screen_size_function"] = lambda stdscr: (window_height, window_width)
+            OptionPicker = Picker(submenu_win, **option_picker_data)
+            s, o, f = OptionPicker.run()
+
+            if o == "refresh": 
+                self.draw_screen()
+                continue
+            if s:
+                selected_columns = s
+                self.hidden_columns = [i for i in range(len(self.column_widths)) if i not in selected_columns]
+
+                # return {x: options[x] for x in s}, o, f
+                break
+            return {}, "", f
 
     def notification(self, stdscr: curses.window, message: str="", title:str="Notification", colours_end: int=0, duration:int=4) -> None:
 
@@ -1729,7 +1909,6 @@ class Picker:
                 "crosshair_cursor": False,
                 "show_header": False,
                 "screen_size_function": lambda stdscr: (notification_height, notification_width),
-
             }
             OptionPicker = Picker(submenu_win, **notification_data)
             s, o, f = OptionPicker.run()
@@ -1933,6 +2112,9 @@ class Picker:
                         self.draw_screen()
                         self.notification(self.stdscr, message=f"Theme {self.colour_theme_number} applied.")
                     self.colours = get_colours(self.colour_theme_number)
+                elif setting == "colsel":
+                    self.draw_screen()
+                    self.select_columns(self.stdscr)
 
                 else:
                     self.user_settings = ""
@@ -2298,7 +2480,13 @@ class Picker:
         self.logger.info(f"function: fetch_data()")
         tmp_items, tmp_header = [], []
         self.getting_data.clear()
-        self.refresh_function(tmp_items, tmp_header, self.visible_rows_indices, self.getting_data)
+        self.refresh_function(
+            tmp_items, 
+            tmp_header, 
+            self.visible_rows_indices, 
+            self.getting_data,
+            self.get_function_data(),
+        )
         if self.track_entries_upon_refresh:
             selected_indices = get_selected_indices(self.selections)
             self.ids = [item[self.id_column] for i, item in enumerate(self.items) if i in selected_indices]
@@ -2396,10 +2584,6 @@ class Picker:
             row_len = 1
             if self.header: row_len = len(self.header)
             elif len(self.items): row_len  = len(self.items[0])
-            # if len(self.indexed_items) == 0:
-            #     insert_at_pos = 0
-            # else:
-            #     insert_at_pos = self.indexed_items[self.cursor_pos][0]
             self.items = self.items[:pos] + [["" for x in range(row_len)]] + self.items[pos:]
             if pos <= self.cursor_pos:
                 self.cursor_pos += 1
@@ -2549,18 +2733,45 @@ class Picker:
             max_total_width=self.rows_w,
             unicode_char_width=self.unicode_char_width
         )
+        self.calculate_section_sizes()
 
-        row_width = sum(self.column_widths) + len(self.separator)*(len(self.column_widths)-1)
+        row_width = sum(self.visible_column_widths) + len(self.separator)*(len(self.visible_column_widths)-1)
         if row_width - self.leftmost_char < self.rows_w:
             if row_width <= self.rows_w - self.left_gutter_width:
                 self.leftmost_char = 0
             else:
                 self.leftmost_char = row_width - (self.rows_w - self.left_gutter_width) + 5
 
+    def cleanup_processes(self):
+        self.thread_stop_event.set()
+        self.data_generation_queue.clear()
+        # with self.data_generation_queue.mutex:
+        #     self.data_generation_queue.queue.clear()
+        function_data = self.get_function_data()
+        for proc in self.processes:
+            if proc.is_alive():
+                proc.terminate()
+                proc.join(timeout=0.01)
+        self.processes = []
+        self.items_sync_loop_event.set()
+        if self.items_sync_thread != None:
+            self.items_sync_thread.join(timeout=1)
+
+    def cleanup_threads(self):
+        self.thread_stop_event.set()
+        with self.data_generation_queue.mutex:
+            self.data_generation_queue.queue.clear()
+        function_data = self.get_function_data()
+        for t in self.threads:
+            if t.is_alive():
+                t.join(timeout=0.01)
+
     def run(self) -> Tuple[list[int], str, dict]:
         """ Run the picker. """
         self.logger.info(f"function: run()")
 
+        self.thread_stop_event.clear()
+
         if self.get_footer_string_startup and self.footer_string_refresh_function != None:
             self.footer_string = " "
             self.footer.adjust_sizes(self.term_h, self.term_w)
@@ -2665,9 +2876,12 @@ class Picker:
 
             elif self.check_key("refresh", key, self.keys_dict) or self.remapped_key(key, curses.KEY_F5, self.key_remappings) or (self.auto_refresh and (time.time() - self.initial_time) >= self.timer):
                 self.logger.debug(f"Get new data (refresh).")
-                self.stdscr.addstr(0,self.term_w-3,"  ", curses.color_pair(self.colours_start+21) | curses.A_BOLD)
+                try:
+                    self.stdscr.addstr(0,self.term_w-3,"  ", curses.color_pair(self.colours_start+21) | curses.A_BOLD)
+                except:
+                    pass
                 self.stdscr.refresh()
-                if self.get_new_data and self.refresh_function:
+                if self.get_new_data:
                     self.refreshing_data = True
 
                     t = threading.Thread(target=self.fetch_data)
@@ -2810,6 +3024,14 @@ class Picker:
                 data["option_functions"] = f"[...] length = {len(data['option_functions'])}"
                 data["loaded_file_states"] = f"[...] length = {len(data['loaded_file_states'])}"
                 data["sheet_states"] = f"[...] length = {len(data['sheet_states'])}"
+                data["highlights"] = f"[...] length = {len(data['highlights'])}"
+                data["colours"] = f"[...] length = {len(data['colours'])}"
+                data["keys_dict"] = f"[...] length = {len(data['keys_dict'])}"
+                data["history_filter_and_search"] = f"[...] length = {len(data['history_filter_and_search'])}"
+                data["history_opts"] = f"[...] length = {len(data['history_opts'])}"
+                data["history_edits"] = f"[...] length = {len(data['history_edits'])}"
+                data["history_pipes"] = f"[...] length = {len(data['history_pipes'])}"
+                data["history_settings"] = f"[...] length = {len(data['history_settings'])}"
                 info_items += [
                     ["",""],
                     ["  get_function_data()", "-*"*30],
@@ -2855,6 +3077,7 @@ class Picker:
             elif self.check_key("exit", key, self.keys_dict):
                 self.stdscr.clear()
                 if len(self.loaded_files) <= 1:
+                    self.cleanup_threads()
                     function_data = self.get_function_data()
                     restore_terminal_settings(tty_fd, self.saved_terminal_state)
                     return [], "", function_data
@@ -2876,6 +3099,7 @@ class Picker:
                     self.draw_screen()
 
             elif self.check_key("full_exit", key, self.keys_dict):
+                self.cleanup_threads()
                 close_curses(self.stdscr)
                 restore_terminal_settings(tty_fd, self.saved_terminal_state)
                 exit()
@@ -2917,11 +3141,12 @@ class Picker:
                 options = []
                 if len(self.items) > 0:
                     options += [["cv", "Centre rows vertically"]]
-                    options += [["pc", "Pin cursor to row number when data refreshes"]]
+                    options += [["pc", "Pin cursor to row index during data refresh."]]
                     options += [["ct", "Centre column-set in terminal"]]
                     options += [["cc", "Centre values in cells"]]
                     options += [["!r", "Toggle auto-refresh"]]
                     options += [["th", "Cycle between themes. (accepts th#)"]]
+                    options += [["colsel", "Toggle columns."]]
                     options += [["nohl", "Toggle highlights"]]
                     options += [["footer", "Toggle footer"]]
                     options += [["header", "Toggle header"]]
@@ -3052,16 +3277,13 @@ class Picker:
                     self.cursor_pos = new_pos
                 self.ensure_no_overscroll()
                 self.draw_screen()
-                # current_row = items_per_page - 1
-                # if current_page + 1 == (len(self.indexed_items) + items_per_page - 1) // items_per_page:
-                #
-                #     current_row = (len(self.indexed_items) +items_per_page - 1) % items_per_page
-                # self.draw_screen()
+
             elif self.check_key("enter", key, self.keys_dict):
                 self.logger.info(f"key_function enter")
                 # Print the selected indices if any, otherwise print the current index
                 if self.is_selecting or self.is_deselecting: self.handle_visual_selection()
                 if len(self.items) == 0:
+                    self.cleanup_threads()
                     function_data = self.get_function_data()
                     restore_terminal_settings(tty_fd, self.saved_terminal_state)
                     return [], "", function_data
@@ -3088,6 +3310,7 @@ class Picker:
                             )
 
                 if options_sufficient:
+                    self.cleanup_threads()
                     self.user_opts = usrtxt
                     self.stdscr.clear()
                     self.stdscr.refresh()
@@ -3101,15 +3324,7 @@ class Picker:
                 self.cursor_pos = max(0, self.cursor_pos-self.items_per_page)
 
             elif self.check_key("redraw_screen", key, self.keys_dict):
-                self.logger.info(f"key_function redraw_screen")
-                self.stdscr.clear()
-                self.stdscr.refresh()
-                restrict_curses(self.stdscr)
-                unrestrict_curses(self.stdscr)
-                self.stdscr.clear()
-                self.stdscr.refresh()
-
-                self.draw_screen()
+                self.refresh_and_draw_screen()
 
             elif self.check_key("cycle_sort_method", key, self.keys_dict):
                 if self.sort_column == self.selected_column:
@@ -3150,22 +3365,28 @@ class Picker:
                         sort_items(self.indexed_items, sort_method=self.columns_sort_method[self.sort_column], sort_column=self.sort_column, sort_reverse=self.sort_reverse[self.sort_column])  # Re-sort self.items based on new column
                         self.cursor_pos = [row[0] for row in self.indexed_items].index(current_index)
             elif self.check_key("col_select_next", key, self.keys_dict):
-                if len(self.items) > 0 and len(self.items[0]) > 0:
-                    col_index = (self.selected_column +1) % (len(self.items[0]))
-                    self.selected_column = col_index
-                # Flash when we loop back to the first column
-                # if self.selected_column == 0:
-                #     curses.flash()
                 self.logger.info(f"key_function col_select_next {self.selected_column}")
+                if len(self.hidden_columns) != len(self.column_widths):
+                    if len(self.items) > 0 and len(self.items[0]) > 0:
+                        while True:
+                            self.hidden_columns
+                            col_index = (self.selected_column +1) % (len(self.items[0]))
+                            self.selected_column = col_index
+                            if self.selected_column not in self.hidden_columns:
+                                break
+
+                    # Flash when we loop back to the first column
+                    # if self.selected_column == 0:
+                    #     curses.flash()
 
 
                 ## Scroll with column select
                 self.get_visible_rows()
                 self.column_widths = get_column_widths(self.visible_rows, header=self.header, max_column_width=self.max_column_width, number_columns=self.number_columns, max_total_width=self.rows_w, unicode_char_width=self.unicode_char_width)
-                visible_column_widths = [c for i,c in enumerate(self.column_widths) if i not in self.hidden_columns]
-                column_set_width = sum(visible_column_widths)+len(self.separator)*len(visible_column_widths)
-                start_of_cell = sum(visible_column_widths[:self.selected_column])+len(self.separator)*self.selected_column
-                end_of_cell = sum(visible_column_widths[:self.selected_column+1])+len(self.separator)*(self.selected_column+1)
+                self.visible_column_widths = [c for i,c in enumerate(self.column_widths) if i not in self.hidden_columns]
+                column_set_width = sum(self.visible_column_widths)+len(self.separator)*len(self.visible_column_widths)
+                start_of_cell = sum(self.visible_column_widths[:self.selected_column])+len(self.separator)*self.selected_column
+                end_of_cell = sum(self.visible_column_widths[:self.selected_column+1])+len(self.separator)*(self.selected_column+1)
                 display_width = self.rows_w-self.left_gutter_width
                 # If the full column is within the current display then don't do anything
                 if start_of_cell >= self.leftmost_char and end_of_cell <= self.leftmost_char + display_width:
@@ -3178,11 +3399,17 @@ class Picker:
                 self.ensure_no_overscroll()
 
             elif self.check_key("col_select_prev", key, self.keys_dict):
-                if len(self.items) > 0 and len(self.items[0]) > 0:
-                    col_index = (self.selected_column -1) % (len(self.items[0]))
-                    self.selected_column = col_index
-
                 self.logger.info(f"key_function col_select_prev {self.selected_column}")
+
+                if len(self.hidden_columns) != len(self.column_widths):
+                    if len(self.items) > 0 and len(self.items[0]) > 0:
+                        while True:
+                            self.hidden_columns
+                            col_index = (self.selected_column -1) % (len(self.items[0]))
+                            self.selected_column = col_index
+                            if self.selected_column not in self.hidden_columns:
+                                break
+
                 # Flash when we loop back to the last column
                 # if self.selected_column == len(self.column_widths)-1:
                 #     curses.flash()
@@ -3190,10 +3417,10 @@ class Picker:
                 ## Scroll with column select
                 self.get_visible_rows()
                 self.column_widths = get_column_widths(self.visible_rows, header=self.header, max_column_width=self.max_column_width, number_columns=self.number_columns, max_total_width=self.rows_w, unicode_char_width=self.unicode_char_width)
-                visible_column_widths = [c for i,c in enumerate(self.column_widths) if i not in self.hidden_columns]
-                column_set_width = sum(visible_column_widths)+len(self.separator)*len(visible_column_widths)
-                start_of_cell = sum(visible_column_widths[:self.selected_column])+len(self.separator)*self.selected_column
-                end_of_cell = sum(visible_column_widths[:self.selected_column+1])+len(self.separator)*(self.selected_column+1)
+                self.visible_column_widths = [c for i,c in enumerate(self.column_widths) if i not in self.hidden_columns]
+                column_set_width = sum(self.visible_column_widths)+len(self.separator)*len(self.visible_column_widths)
+                start_of_cell = sum(self.visible_column_widths[:self.selected_column])+len(self.separator)*self.selected_column
+                end_of_cell = sum(self.visible_column_widths[:self.selected_column+1])+len(self.separator)*(self.selected_column+1)
                 display_width = self.rows_w-self.left_gutter_width
 
                 # If the entire column is within the current display then don't do anything
@@ -3209,21 +3436,21 @@ class Picker:
             elif self.check_key("scroll_right", key, self.keys_dict):
                 self.logger.info(f"key_function scroll_right")
                 if len(self.indexed_items):
-                    row_width = sum(self.column_widths) + len(self.separator)*(len(self.column_widths)-1)
+                    row_width = sum(self.visible_column_widths) + len(self.separator)*(len(self.visible_column_widths)-1)
                     if row_width-self.leftmost_char >= self.rows_w-5:
                         self.leftmost_char += 5
                     self.leftmost_char = min(self.leftmost_char, row_width - (self.rows_w) + self.left_gutter_width+5)
-                if sum(self.column_widths) + len(self.column_widths)*len(self.separator) < self.rows_w:
+                if sum(self.visible_column_widths) + len(self.visible_column_widths)*len(self.separator) < self.rows_w:
                     self.leftmost_char = 0
 
             elif self.check_key("scroll_right_25", key, self.keys_dict):
                 self.logger.info(f"key_function scroll_right")
                 if len(self.indexed_items):
-                    row_width = sum(self.column_widths) + len(self.separator)*(len(self.column_widths)-1)
+                    row_width = sum(self.visible_column_widths) + len(self.separator)*(len(self.visible_column_widths)-1)
                     if row_width-self.leftmost_char+5 >= self.rows_w-25:
                         self.leftmost_char += 25
                     self.leftmost_char = min(self.leftmost_char, row_width - (self.rows_w) + self.left_gutter_width+5)
-                if sum(self.column_widths) + len(self.column_widths)*len(self.separator) < self.rows_w:
+                if sum(self.visible_column_widths) + len(self.visible_column_widths)*len(self.separator) < self.rows_w:
                     self.leftmost_char = 0
 
             elif self.check_key("scroll_left", key, self.keys_dict):
@@ -3242,24 +3469,13 @@ class Picker:
             elif self.check_key("scroll_far_right", key, self.keys_dict):
                 self.logger.info(f"key_function scroll_far_right")
                 longest_row_str_len = 0
-                # rows = self.get_visible_rows()
-                # for i in range(len(rows)):
-                #     item = rows[i]
-                #     row_str = format_row(item, self.hidden_columns, self.column_widths, self.separator, self.centre_in_cols, self.unicode_char_width)
-                #     if len(row_str) > longest_row_str_len: longest_row_str_len=len(row_str)
-                longest_row_str_len = sum(self.column_widths) + (len(self.column_widths)-1)*len(self.separator)
-                # for i in range(len(self.indexed_items)):
-                #     item = self.indexed_items[i]
-                #     row_str = format_row(item[1], self.hidden_columns, self.column_widths, self.separator, self.centre_in_cols)
-                #     if len(row_str) > longest_row_str_len: longest_row_str_len=len(row_str)
-                # self.notification(self.stdscr, f"{longest_row_str_len}")
+                longest_row_str_len = sum(self.visible_column_widths) + (len(self.visible_column_widths)-1)*len(self.separator)
                 if len(self.indexed_items):
-                    row_width = sum(self.column_widths) + len(self.separator)*(len(self.column_widths)-1)
+                    row_width = sum(self.visible_column_widths) + len(self.separator)*(len(self.visible_column_widths)-1)
                     self.leftmost_char = row_width - (self.rows_w) + self.left_gutter_width+5
                 self.leftmost_char = min(self.leftmost_char, row_width - (self.rows_w) + self.left_gutter_width+5)
 
-                longest_row_str_len = sum(self.column_widths) + (len(self.column_widths)-1)*len(self.separator)
-                # self.leftmost_char = max(0, longest_row_str_len-self.rows_w+2+5)
+                longest_row_str_len = sum(self.visible_column_widths) + (len(self.visible_column_widths)-1)*len(self.separator)
 
 
 
@@ -3321,16 +3537,6 @@ class Picker:
                     self.selected_column = min(self.selected_column, row_len-2)
                 self.initialise_variables()
 
-
-
-
-            # elif self.check_key("increase_lines_per_page", key, self.keys_dict):
-            #     self.items_per_page += 1
-            #     self.draw_screen()
-            # elif self.check_key("decrease_lines_per_page", key, self.keys_dict):
-            #     if self.items_per_page > 1:
-            #         self.items_per_page -= 1
-            #     self.draw_screen()
             elif self.check_key("decrease_column_width", key, self.keys_dict):
                 self.logger.info(f"key_function decrease_column_width")
                 if self.max_column_width > 10:
@@ -3512,12 +3718,6 @@ class Picker:
                     function_data = self.get_function_data()
                     return [], "escape", function_data
 
-
-                # else:
-                #     self.search_query = ""
-                #     self.mode_index = 0
-                #     self.highlights = [highlight for highlight in self.highlights if "type" not in highlight or highlight["type"] != "search" ]
-                #     continue
                 self.draw_screen()
 
             elif self.check_key("opts_input", key, self.keys_dict):
@@ -3733,7 +3933,7 @@ class Picker:
 
             elif self.check_key("edit", key, self.keys_dict):
                 self.logger.info(f"key_function edit")
-                if len(self.indexed_items) > 0 and self.selected_column >=0 and self.editable_columns[self.selected_column]:
+                if len(self.indexed_items) > 0 and self.editable_columns[self.selected_column]:
                     current_val = self.indexed_items[self.cursor_pos][1][self.selected_column]
                     usrtxt = f"{current_val}"
                     field_end_f = lambda: self.get_term_size()[1]-38 if self.show_footer else lambda: self.get_term_size()[1]-3
@@ -3762,6 +3962,47 @@ class Picker:
                             usrtxt = str(eval(usrtxt[3:]))
                         self.indexed_items[self.cursor_pos][1][self.selected_column] = usrtxt
                         self.history_edits.append(usrtxt)
+            elif self.check_key("edit_nvim", key, self.keys_dict):
+
+                def edit_strings_in_nvim(strings: list[str]) -> list[str]:
+                    """
+                    Opens a list of strings in nvim for editing and returns the modified strings.
+
+                    Args:
+                        strings (list[str]): The list of strings to edit.
+
+                    Returns:
+                        list[str]: The updated list of strings after editing in nvim.
+                    """
+
+                    # Open the strings in a tmpfile for editing
+                    with tempfile.NamedTemporaryFile(mode="w+", suffix=".txt", delete=False) as tmp:
+                        tmp.write("\n".join(strings))
+                        tmp.flush()
+                        tmp_name = tmp.name
+
+                    subprocess.run(["nvim", tmp_name])
+
+                    # Read the modified strings into a list and return them.
+                    with open(tmp_name, "r") as tmp:
+                        edited_content = tmp.read().splitlines()
+
+                    return edited_content
+
+                if len(self.indexed_items) > 0 and self.editable_columns[self.selected_column]:
+
+                    selected_cells = [self.items[index][self.selected_column] for index, selected in self.selections.items() if selected ]
+                    selected_cells_indices = [(index, self.selected_column) for index, selected in self.selections.items() if selected ]
+
+                    edited_cells = edit_strings_in_nvim(selected_cells)
+                    count = 0
+                    if len(edited_cells) == len(selected_cells_indices):
+                        for i, j in selected_cells_indices:
+                            self.items[i][j] = edited_cells[count]
+                            count += 1
+
+                    self.refresh_and_draw_screen()
+
 
             elif self.check_key("edit_picker", key, self.keys_dict):
                 self.logger.info(f"key_function edit_picker")
@@ -3793,7 +4034,7 @@ class Picker:
                         self.indexed_items[self.cursor_pos][1][self.selected_column] = usrtxt
                         self.history_edits.append(usrtxt)
             elif self.check_key("edit_ipython", key, self.keys_dict):
-                self.logger.info(f"key_function edit_picker")
+                self.logger.info(f"key_function edit_ipython")
                 import IPython, termios
                 self.stdscr.clear()
                 restrict_curses(self.stdscr)
@@ -3937,7 +4178,7 @@ def parse_arguments() -> Tuple[argparse.Namespace, dict]:
     #     input_arg = args.filename
 
     elif args.generate:
-        function_data["refresh_function"] = lambda items, header, visible_rows_indices, getting_data: generate_picker_data_from_file(args.generate, items, header, visible_rows_indices, getting_data)
+        function_data["refresh_function"] = lambda items, header, visible_rows_indices, getting_data, state: generate_picker_data_from_file(args.generate, items, header, visible_rows_indices, getting_data, state)
         function_data["get_data_startup"] = True
         function_data["get_new_data"] = True
         return args, function_data
@@ -4197,7 +4438,7 @@ def main() -> None:
     # function_data["debug"] = True
     # function_data["debug_level"] = 1
 
-    function_data["cell_cursor"] = False
+    # function_data["cell_cursor"] = False
 
     function_data["split_right"] = False
     function_data["split_left"] = False