cjm-fasthtml-token-selector 0.0.1__py3-none-any.whl

cjm_fasthtml_token_selector/helpers/tokenizer.py

@@ -0,0 +1,76 @@
+"""Tokenization utilities for splitting text into tokens and converting between token indices and character positions."""
+
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/helpers/tokenizer.ipynb.
+
+# %% auto #0
+__all__ = ['count_tokens', 'get_token_list', 'token_index_to_char_position', 'tokenize']
+
+# %% ../../nbs/helpers/tokenizer.ipynb #c161bb62
+from typing import Any, List, Optional, Union
+
+from ..core.models import Token
+
+# %% ../../nbs/helpers/tokenizer.ipynb #1cded653
+def count_tokens(
+    text:str,  # text to count tokens in
+) -> int:  # token count
+    """Count the number of whitespace-delimited tokens in text."""
+    if not text:
+        return 0
+    return len(text.split())
+
+# %% ../../nbs/helpers/tokenizer.ipynb #71f880db
+def get_token_list(
+    text:str,  # text to split into tokens
+) -> List[str]:  # list of token strings
+    """Split text into a list of whitespace-delimited tokens."""
+    if not text:
+        return []
+    return text.split()
+
+# %% ../../nbs/helpers/tokenizer.ipynb #b0f6d4c0
+def token_index_to_char_position(
+    text:str,         # full text string
+    token_index:int,  # 0-based token index
+) -> int:  # character position for split
+    """Convert a token index to the character position where a split should occur."""
+    if token_index <= 0:
+        return 0
+
+    words = text.split()
+    if token_index >= len(words):
+        return len(text)
+
+    # Walk through the original text to find the character position
+    # before the word at token_index
+    pos = 0
+    for i, word in enumerate(words):
+        if i == token_index:
+            break
+        pos += len(word)
+        # Advance past whitespace between words
+        while pos < len(text) and text[pos] == ' ':
+            pos += 1
+
+    return pos
+
+# %% ../../nbs/helpers/tokenizer.ipynb #c4c9d469
+def tokenize(
+    text_or_tokens:Union[str, List[str]],  # raw text string or pre-tokenized list
+    metadata:Optional[List[Any]] = None,   # per-token metadata (must match token count)
+) -> List[Token]:  # list of Token objects
+    """Convert text or a pre-tokenized list into Token objects."""
+    if isinstance(text_or_tokens, str):
+        words = get_token_list(text_or_tokens)
+    else:
+        words = list(text_or_tokens)
+
+    if metadata is not None and len(metadata) != len(words):
+        raise ValueError(
+            f"metadata length ({len(metadata)}) must match token count ({len(words)})"
+        )
+
+    return [
+        Token(text=w, index=i, metadata=metadata[i] if metadata else None)
+        for i, w in enumerate(words)
+    ]

cjm_fasthtml_token_selector/js/core.py

@@ -0,0 +1,181 @@
+"""Master IIFE composer for the token selector JS runtime."""
+
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/js/core.ipynb.
+
+# %% auto #0
+__all__ = ['global_callback_name', 'generate_token_selector_js']
+
+# %% ../../nbs/js/core.ipynb #3dd234c7
+from typing import Any, Tuple
+
+from fasthtml.common import Script
+
+from ..core.config import TokenSelectorConfig
+from ..core.models import TokenSelectorState
+from ..core.html_ids import TokenSelectorHtmlIds
+from .navigation import generate_navigation_js
+from .display import generate_display_js
+from .repeat import generate_key_repeat_js
+
+# %% ../../nbs/js/core.ipynb #bee1f126
+_GLOBAL_CALLBACKS = (
+    "activate",
+    "deactivate",
+    "selectGap",
+    "selectWord",
+    "selectToken",
+    "moveToFirst",
+    "moveToLast",
+)
+
+def global_callback_name(
+    prefix:str,    # token selector instance prefix
+    callback:str,  # base callback name
+) -> str:  # global function name
+    """Generate a prefix-unique global callback name."""
+    return f"{prefix}_{callback}"
+
+# %% ../../nbs/js/core.ipynb #173e8f33
+def _generate_state_init_js(
+    config:TokenSelectorConfig,  # config for this instance
+    state:TokenSelectorState,    # initial state
+) -> str:  # JS code fragment
+    """Generate state initialization code."""
+    return f"""
+    // State
+    ns.anchor = {state.anchor};
+    ns.focus = {state.focus};
+    ns.wordCount = {state.word_count};
+    ns.active = false;
+    ns.mode = '{config.selection_mode}';
+    ns._prevAnchor = -1;
+"""
+
+# %% ../../nbs/js/core.ipynb #94cc4f28
+def _generate_on_change_js(
+    config:TokenSelectorConfig,  # config for this instance
+) -> str:  # JS code fragment
+    """Generate the on-change callback dispatcher."""
+    if config.on_change_callback:
+        return f"""
+    ns._fireOnChange = function() {{
+        if (typeof window['{config.on_change_callback}'] === 'function') {{
+            window['{config.on_change_callback}'](ns.anchor, ns.focus, ns.mode);
+        }}
+    }};
+"""
+    return """
+    ns._fireOnChange = function() {};
+"""
+
+# %% ../../nbs/js/core.ipynb #0089ec56
+def _generate_activation_js(
+    config:TokenSelectorConfig,  # config for this instance
+    ids:TokenSelectorHtmlIds,    # HTML IDs
+) -> str:  # JS code fragment
+    """Generate activate/deactivate functions."""
+    return f"""
+    ns.activate = function() {{
+        ns.active = true;
+        // Re-read word count from DOM
+        var grid = document.getElementById('{ids.token_grid}');
+        if (grid) ns.wordCount = parseInt(grid.dataset.wordCount) || 0;
+        // Reset position
+        ns.anchor = 0;
+        ns.focus = 0;
+        // Add key listeners
+        document.addEventListener('keydown', ns._handleKeyDown, true);
+        document.addEventListener('keyup', ns._handleKeyUp, true);
+        ns.updateDisplay();
+        ns.updateInputs();
+    }};
+
+    ns.deactivate = function() {{
+        ns.active = false;
+        // Cancel repeat
+        if (ns._repeatTimer) {{
+            clearTimeout(ns._repeatTimer);
+            ns._repeatTimer = null;
+        }}
+        ns._repeatKey = null;
+        ns._repeatShift = false;
+        // Remove key listeners
+        document.removeEventListener('keydown', ns._handleKeyDown, true);
+        document.removeEventListener('keyup', ns._handleKeyUp, true);
+    }};
+"""
+
+# %% ../../nbs/js/core.ipynb #d62a060d
+def _generate_global_callbacks_js(
+    config:TokenSelectorConfig,  # config for this instance
+) -> str:  # JS code fragment
+    """Generate global callback wrappers."""
+    lines = []
+    for cb_name in _GLOBAL_CALLBACKS:
+        global_name = global_callback_name(config.prefix, cb_name)
+        lines.append(
+            f"    window['{global_name}'] = function() {{ "
+            f"if (ns.{cb_name}) ns.{cb_name}.apply(ns, arguments); }};"
+        )
+    return "\n".join(lines)
+
+# %% ../../nbs/js/core.ipynb #111900aa
+def _generate_settle_handler_js(
+    config:TokenSelectorConfig,  # config for this instance
+    ids:TokenSelectorHtmlIds,    # HTML IDs
+) -> str:  # JS code fragment
+    """Generate htmx:afterSettle handler for swap resilience."""
+    guard = f"_tsMasterListener_{config.prefix}"
+    return f"""
+    // HTMX afterSettle: re-read word count and update display
+    if (!window['{guard}']) {{
+        window['{guard}'] = true;
+        document.body.addEventListener('htmx:afterSettle', function() {{
+            var grid = document.getElementById('{ids.token_grid}');
+            if (grid && ns.active) {{
+                ns.wordCount = parseInt(grid.dataset.wordCount) || 0;
+                ns.updateDisplay();
+                ns.updateInputs();
+            }}
+        }});
+    }}
+"""
+
+# %% ../../nbs/js/core.ipynb #48c0e44b
+def generate_token_selector_js(
+    config:TokenSelectorConfig,              # config for this instance
+    ids:TokenSelectorHtmlIds,                # HTML IDs
+    state:TokenSelectorState = None,         # initial state
+    extra_scripts:Tuple[str, ...] = (),      # additional JS to include in the IIFE
+) -> Any:  # Script element with the complete IIFE
+    """Compose all token selector JS into a single namespaced IIFE."""
+    if state is None:
+        state = TokenSelectorState()
+
+    # Collect all JS fragments
+    fragments = [
+        _generate_state_init_js(config, state),
+        _generate_on_change_js(config),
+        generate_display_js(config, ids),
+        generate_navigation_js(config, ids),
+        generate_key_repeat_js(config),
+        _generate_activation_js(config, ids),
+        _generate_global_callbacks_js(config),
+        _generate_settle_handler_js(config, ids),
+    ]
+
+    # Consumer extra scripts
+    if extra_scripts:
+        fragments.append("\n".join(extra_scripts))
+
+    body = "\n".join(fragments)
+    prefix = config.prefix
+
+    iife = f"""(function() {{
+    'use strict';
+    window.tokenSelectors = window.tokenSelectors || {{}};
+    var ns = window.tokenSelectors['{prefix}'] = {{}};
+{body}
+}})();"""
+
+    return Script(iife)

cjm_fasthtml_token_selector/js/display.py

@@ -0,0 +1,146 @@
+"""Generates JS functions for updating token display state (caret indicators, highlights, dimming, hidden inputs)."""
+
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/js/display.ipynb.
+
+# %% auto #0
+__all__ = ['generate_display_js']
+
+# %% ../../nbs/js/display.ipynb #4614e222
+from cjm_fasthtml_token_selector.core.constants import (
+    CARET_INDICATOR_CLS, OPACITY_50_CLS, HIGHLIGHT_CLS,
+)
+from ..core.config import TokenSelectorConfig
+from ..core.html_ids import TokenSelectorHtmlIds
+
+# %% ../../nbs/js/display.ipynb #85e3702e
+def _generate_update_inputs_js(
+    ids:TokenSelectorHtmlIds,  # HTML IDs
+) -> str:  # JS code fragment
+    """Generate the hidden input sync function."""
+    return f"""
+    ns.updateInputs = function() {{
+        var anchorEl = document.getElementById('{ids.anchor_input}');
+        var focusEl = document.getElementById('{ids.focus_input}');
+        if (anchorEl) anchorEl.value = ns.anchor;
+        if (focusEl) focusEl.value = ns.focus;
+    }};
+"""
+
+# %% ../../nbs/js/display.ipynb #9c85ab30
+def _generate_gap_display_js(
+    ids:TokenSelectorHtmlIds,  # HTML IDs
+) -> str:  # JS code fragment
+    """Generate gap mode display update function."""
+    return f"""
+    ns.updateDisplay = function() {{
+        var grid = document.getElementById('{ids.token_grid}');
+        if (!grid) return;
+
+        // Remove existing caret indicators
+        grid.querySelectorAll('.caret-indicator').forEach(function(el) {{ el.remove(); }});
+
+        var tokens = grid.querySelectorAll('.token');
+        var prevAnchor = ns._prevAnchor !== undefined ? ns._prevAnchor : -1;
+
+        tokens.forEach(function(token) {{
+            var idx = parseInt(token.dataset.tokenIndex);
+
+            // Dimming: tokens before caret
+            if (idx < ns.anchor) {{
+                if (!token.classList.contains('{OPACITY_50_CLS}')) {{
+                    token.classList.add('{OPACITY_50_CLS}');
+                }}
+            }} else {{
+                token.classList.remove('{OPACITY_50_CLS}');
+            }}
+
+            // Caret indicator at anchor position
+            if (idx === ns.anchor) {{
+                var caret = document.createElement('div');
+                caret.className = '{CARET_INDICATOR_CLS}';
+                token.insertBefore(caret, token.firstChild);
+            }}
+        }});
+
+        ns._prevAnchor = ns.anchor;
+    }};
+"""
+
+# %% ../../nbs/js/display.ipynb #90b62704
+def _generate_word_display_js(
+    ids:TokenSelectorHtmlIds,  # HTML IDs
+) -> str:  # JS code fragment
+    """Generate word mode display update function."""
+    return f"""
+    ns.updateDisplay = function() {{
+        var grid = document.getElementById('{ids.token_grid}');
+        if (!grid) return;
+
+        var tokens = grid.querySelectorAll('.token');
+
+        tokens.forEach(function(token) {{
+            var idx = parseInt(token.dataset.tokenIndex);
+
+            if (idx === ns.anchor) {{
+                if (!token.classList.contains('{HIGHLIGHT_CLS}')) {{
+                    token.classList.add('{HIGHLIGHT_CLS}');
+                }}
+            }} else {{
+                token.classList.remove('{HIGHLIGHT_CLS}');
+            }}
+        }});
+    }};
+"""
+
+# %% ../../nbs/js/display.ipynb #f3f54861
+def _generate_span_display_js(
+    ids:TokenSelectorHtmlIds,  # HTML IDs
+) -> str:  # JS code fragment
+    """Generate span mode display update function."""
+    return f"""
+    ns.updateDisplay = function() {{
+        var grid = document.getElementById('{ids.token_grid}');
+        if (!grid) return;
+
+        // Remove existing caret indicators
+        grid.querySelectorAll('.caret-indicator').forEach(function(el) {{ el.remove(); }});
+
+        var tokens = grid.querySelectorAll('.token');
+        var lo = Math.min(ns.anchor, ns.focus);
+        var hi = Math.max(ns.anchor, ns.focus);
+
+        tokens.forEach(function(token) {{
+            var idx = parseInt(token.dataset.tokenIndex);
+
+            // Highlight tokens in range
+            if (idx >= lo && idx <= hi) {{
+                if (!token.classList.contains('{HIGHLIGHT_CLS}')) {{
+                    token.classList.add('{HIGHLIGHT_CLS}');
+                }}
+            }} else {{
+                token.classList.remove('{HIGHLIGHT_CLS}');
+            }}
+
+            // Caret at focus position
+            if (idx === ns.focus) {{
+                var caret = document.createElement('div');
+                caret.className = '{CARET_INDICATOR_CLS}';
+                token.insertBefore(caret, token.firstChild);
+            }}
+        }});
+    }};
+"""
+
+# %% ../../nbs/js/display.ipynb #c782dde5
+def generate_display_js(
+    config:TokenSelectorConfig,  # config for this instance
+    ids:TokenSelectorHtmlIds,    # HTML IDs
+) -> str:  # JS code fragment for the IIFE
+    """Generate display update and hidden input sync JS functions."""
+    display_generators = {
+        "gap": _generate_gap_display_js,
+        "word": _generate_word_display_js,
+        "span": _generate_span_display_js,
+    }
+    gen = display_generators.get(config.selection_mode, _generate_gap_display_js)
+    return _generate_update_inputs_js(ids) + gen(ids)

cjm_fasthtml_token_selector/js/navigation.py

@@ -0,0 +1,228 @@
+"""Generates mode-specific navigation and selection JS functions."""
+
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/js/navigation.ipynb.
+
+# %% auto #0
+__all__ = ['generate_navigation_js']
+
+# %% ../../nbs/js/navigation.ipynb #386f20ca
+from ..core.config import TokenSelectorConfig
+from ..core.html_ids import TokenSelectorHtmlIds
+
+# %% ../../nbs/js/navigation.ipynb #7146756c
+def _generate_gap_nav_js(
+    config:TokenSelectorConfig,  # config for this instance
+    ids:TokenSelectorHtmlIds,    # HTML IDs
+) -> str:  # JS code fragment
+    """Generate gap mode navigation functions."""
+    wrap = "true" if config.wrap_navigation else "false"
+    return f"""
+    ns.moveLeft = function() {{
+        if (ns.anchor > 0) {{
+            ns.anchor--;
+            ns.focus = ns.anchor;
+        }} else if ({wrap}) {{
+            ns.anchor = ns.wordCount;
+            ns.focus = ns.anchor;
+        }}
+        ns.updateDisplay();
+        ns.updateInputs();
+        ns._fireOnChange();
+    }};
+
+    ns.moveRight = function() {{
+        if (ns.anchor < ns.wordCount) {{
+            ns.anchor++;
+            ns.focus = ns.anchor;
+        }} else if ({wrap}) {{
+            ns.anchor = 0;
+            ns.focus = ns.anchor;
+        }}
+        ns.updateDisplay();
+        ns.updateInputs();
+        ns._fireOnChange();
+    }};
+
+    ns.selectGap = function(position) {{
+        ns.anchor = Math.max(0, Math.min(position, ns.wordCount));
+        ns.focus = ns.anchor;
+        ns.updateDisplay();
+        ns.updateInputs();
+        ns._fireOnChange();
+    }};
+
+    ns.moveToFirst = function() {{
+        ns.anchor = 0;
+        ns.focus = ns.anchor;
+        ns.updateDisplay();
+        ns.updateInputs();
+        ns._fireOnChange();
+    }};
+
+    ns.moveToLast = function() {{
+        ns.anchor = ns.wordCount;
+        ns.focus = ns.anchor;
+        ns.updateDisplay();
+        ns.updateInputs();
+        ns._fireOnChange();
+    }};
+"""
+
+# %% ../../nbs/js/navigation.ipynb #4ef3a24d
+def _generate_word_nav_js(
+    config:TokenSelectorConfig,  # config for this instance
+    ids:TokenSelectorHtmlIds,    # HTML IDs
+) -> str:  # JS code fragment
+    """Generate word mode navigation functions."""
+    wrap = "true" if config.wrap_navigation else "false"
+    return f"""
+    ns.moveLeft = function() {{
+        if (ns.anchor > 0) {{
+            ns.anchor--;
+            ns.focus = ns.anchor;
+        }} else if ({wrap} && ns.wordCount > 0) {{
+            ns.anchor = ns.wordCount - 1;
+            ns.focus = ns.anchor;
+        }}
+        ns.updateDisplay();
+        ns.updateInputs();
+        ns._fireOnChange();
+    }};
+
+    ns.moveRight = function() {{
+        if (ns.anchor < ns.wordCount - 1) {{
+            ns.anchor++;
+            ns.focus = ns.anchor;
+        }} else if ({wrap}) {{
+            ns.anchor = 0;
+            ns.focus = ns.anchor;
+        }}
+        ns.updateDisplay();
+        ns.updateInputs();
+        ns._fireOnChange();
+    }};
+
+    ns.selectWord = function(index) {{
+        ns.anchor = Math.max(0, Math.min(index, ns.wordCount - 1));
+        ns.focus = ns.anchor;
+        ns.updateDisplay();
+        ns.updateInputs();
+        ns._fireOnChange();
+    }};
+
+    ns.moveToFirst = function() {{
+        ns.anchor = 0;
+        ns.focus = ns.anchor;
+        ns.updateDisplay();
+        ns.updateInputs();
+        ns._fireOnChange();
+    }};
+
+    ns.moveToLast = function() {{
+        ns.anchor = Math.max(0, ns.wordCount - 1);
+        ns.focus = ns.anchor;
+        ns.updateDisplay();
+        ns.updateInputs();
+        ns._fireOnChange();
+    }};
+"""
+
+# %% ../../nbs/js/navigation.ipynb #7a3c234e
+def _generate_span_nav_js(
+    config:TokenSelectorConfig,  # config for this instance
+    ids:TokenSelectorHtmlIds,    # HTML IDs
+) -> str:  # JS code fragment
+    """Generate span mode navigation functions."""
+    wrap = "true" if config.wrap_navigation else "false"
+    return f"""
+    ns.moveLeft = function() {{
+        if (ns.focus > 0) {{
+            ns.focus--;
+            ns.anchor = ns.focus;
+        }} else if ({wrap} && ns.wordCount > 0) {{
+            ns.focus = ns.wordCount - 1;
+            ns.anchor = ns.focus;
+        }}
+        ns.updateDisplay();
+        ns.updateInputs();
+        ns._fireOnChange();
+    }};
+
+    ns.moveRight = function() {{
+        if (ns.focus < ns.wordCount - 1) {{
+            ns.focus++;
+            ns.anchor = ns.focus;
+        }} else if ({wrap}) {{
+            ns.focus = 0;
+            ns.anchor = ns.focus;
+        }}
+        ns.updateDisplay();
+        ns.updateInputs();
+        ns._fireOnChange();
+    }};
+
+    ns.extendLeft = function() {{
+        if (ns.focus > 0) {{
+            ns.focus--;
+        }}
+        ns.updateDisplay();
+        ns.updateInputs();
+        ns._fireOnChange();
+    }};
+
+    ns.extendRight = function() {{
+        if (ns.focus < ns.wordCount - 1) {{
+            ns.focus++;
+        }}
+        ns.updateDisplay();
+        ns.updateInputs();
+        ns._fireOnChange();
+    }};
+
+    ns.selectToken = function(index) {{
+        ns.focus = Math.max(0, Math.min(index, ns.wordCount - 1));
+        ns.updateDisplay();
+        ns.updateInputs();
+        ns._fireOnChange();
+    }};
+
+    ns.selectGap = function(position) {{
+        var idx = Math.max(0, Math.min(position, ns.wordCount - 1));
+        ns.anchor = idx;
+        ns.focus = idx;
+        ns.updateDisplay();
+        ns.updateInputs();
+        ns._fireOnChange();
+    }};
+
+    ns.moveToFirst = function() {{
+        ns.anchor = 0;
+        ns.focus = 0;
+        ns.updateDisplay();
+        ns.updateInputs();
+        ns._fireOnChange();
+    }};
+
+    ns.moveToLast = function() {{
+        var last = Math.max(0, ns.wordCount - 1);
+        ns.anchor = last;
+        ns.focus = last;
+        ns.updateDisplay();
+        ns.updateInputs();
+        ns._fireOnChange();
+    }};
+"""
+
+# %% ../../nbs/js/navigation.ipynb #eb0cbbbb
+def generate_navigation_js(
+    config:TokenSelectorConfig,  # config for this instance
+    ids:TokenSelectorHtmlIds,    # HTML IDs
+) -> str:  # JS code fragment for the IIFE
+    """Generate mode-specific navigation and selection JS functions."""
+    generators = {
+        "gap": _generate_gap_nav_js,
+        "word": _generate_word_nav_js,
+        "span": _generate_span_nav_js,
+    }
+    gen = generators.get(config.selection_mode, _generate_gap_nav_js)
+    return gen(config, ids)