streamdown 0.16.0__py3-none-any.whl → 0.18.0__py3-none-any.whl

streamdown/sd.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env -S uv run --script
+#!/bin/bash
 # /// script
 # requires-python = ">=3.8"
 # dependencies = [
@@ -9,6 +9,13 @@
 #     "toml"
 # ]
 # ///
+'''':
+if command -v uv &> /dev/null; then
+    exec uv run --script "$0" "$@"
+else
+    exec python3 "$0" "$@"
+fi
+'''
 import appdirs, toml
 import logging, tempfile
 import os,      sys
@@ -29,7 +36,7 @@ from functools import reduce
 from argparse import ArgumentParser
 from pygments import highlight
 from pygments.lexers import get_lexer_by_name
-from pygments.formatters import Terminal256Formatter
+from pygments.formatters import TerminalTrueColorFormatter
 from pygments.styles import get_style_by_name
 
 if __package__ is None:
@@ -39,21 +46,23 @@ else:
 
 default_toml = """
 [features]
-CodeSpaces = true
+CodeSpaces = false
 Clipboard  = true
 Logging    = false
-Timeout    = 0.5
+Timeout    = 0.1
+Savebrace  = true
 
 [style]
-Margin      = 2 
-ListIndent  = 2
-PrettyPad   = false
-Width       = 0
+Margin          = 2 
+ListIndent      = 2
+PrettyPad       = false
+PrettyBroken    = true
+Width           = 0
 HSV     = [0.8, 0.5, 0.5]
 Dark    = { H = 1.00, S = 1.50, V = 0.25 }
 Mid     = { H = 1.00, S = 1.00, V = 0.50 }
 Symbol  = { H = 1.00, S = 1.00, V = 1.50 }
-Head    = { H = 1.00, S = 2.00, V = 1.50 }
+Head    = { H = 1.00, S = 1.00, V = 1.75 }
 Grey    = { H = 1.00, S = 0.25, V = 1.37 }
 Bright  = { H = 1.00, S = 2.00, V = 2.00 }
 Syntax  = "monokai"
@@ -90,7 +99,8 @@ ANSIESCAPE = r'\033(?:\[[0-9;?]*[a-zA-Z]|][0-9]*;;.*?\\|\\)'
 KEYCODE_RE = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
 
 visible = lambda x: re.sub(ANSIESCAPE, "", x)
-visible_length = lambda x: len(visible(x))
+# cjk characters are double width
+visible_length = lambda x: len(visible(x)) + cjk_count(x)
 extract_ansi_codes = lambda text: re.findall(ESCAPE, text)
 remove_ansi = lambda line, codeList: reduce(lambda line, code: line.replace(code, ''), codeList, line)
 
@@ -102,6 +112,13 @@ def debug_write(text):
             state.Logging = tempfile.NamedTemporaryFile(dir=tmp_dir, prefix="dbg", delete=False, mode="wb")
         state.Logging.write(text)
 
+def savebrace():
+    if state.Savebrace and state.code_buffer_raw:
+        path = os.path.join(tempfile.gettempdir(), "sd", 'savebrace')
+        with open(path, "a") as f:
+            f.write(state.code_buffer_raw)
+        
+
 class Goto(Exception):
     pass
 
@@ -134,6 +151,7 @@ class ParseState:
         self.Clipboard = _features.get("Clipboard")
         self.Logging = _features.get("Logging")
         self.Timeout = _features.get("Timeout")
+        self.Savebrace = _features.get("Savebrace")
 
         self.WidthArg = None
         self.WidthFull = None
@@ -149,6 +167,7 @@ class ParseState:
         # streaming code blocks while preserving
         # multiline parsing.
         self.code_buffer = ""
+        self.code_buffer_raw = ""
         self.code_gen = 0
         self.code_language = None
         self.code_first_line = False
@@ -157,6 +176,7 @@ class ParseState:
 
         self.ordered_list_numbers = []
         self.list_item_stack = []  # stack of (indent, type)
+        self.list_indent_text = 0
 
         self.in_list = False
         self.in_code = False # (Code.[Backtick|Spaces] | False)
@@ -177,15 +197,22 @@ class ParseState:
         self.where_from = None
 
     def current(self):
-        state = { 'inline': self.inline_code, 'code': self.in_code, 'bold': self.in_bold, 'italic': self.in_italic, 'underline': self.in_underline }
+        state = { 'inline': self.inline_code, 'code': self.in_code, 'bold': self.in_bold, 'italic': self.in_italic, 'underline': self.in_underline, 'strikeout': self.in_strikeout }
         state['none'] = all(item is False for item in state.values())
         return state
 
     def reset_inline(self):
-        self.inline_code = self.in_bold = self.in_italic = self.in_underline = False
+        self.inline_code = self.in_bold = self.in_italic = self.in_underline = self.in_strikeout = False
+
+    def full_width(self, offset = 0):
+        return offset + (state.current_width(listwidth = True) if Style.PrettyBroken else self.WidthFull)
+
+    def current_width(self, listwidth = False):
+        return self.Width - (len(visible(self.space_left(listwidth))) + Style.Margin)
 
-    def space_left(self):
-        return Style.MarginSpaces + (Style.Blockquote * self.block_depth) if len(self.current_line) == 0 else "" 
+    def space_left(self, listwidth = False):
+        pre = ' ' * (len(state.list_item_stack)) * Style.ListIndent if listwidth else ''
+        return pre + Style.MarginSpaces + (Style.Blockquote * self.block_depth) if len(self.current_line) == 0 else "" 
 
 state = ParseState()
 
@@ -196,7 +223,7 @@ def format_table(rowList):
 
     # Calculate max width per column (integer division)
     # Subtract num_cols + 1 for the vertical borders '│'
-    available_width = state.Width - (num_cols + 1)
+    available_width = state.current_width() - (num_cols + 1)
     col_width = max(1, available_width // num_cols)
     bg_color = Style.Mid if state.in_table == Style.Head else Style.Dark
     state.bg = f"{BG}{bg_color}"
@@ -205,7 +232,7 @@ def format_table(rowList):
     # Note this is where every cell is formatted so if 
     # you are styling, do it before here!
     for row in rowList:
-        wrapped_cell = text_wrap(row, width=col_width)
+        wrapped_cell = text_wrap(row, width=col_width, force_truncate=True)
 
         # Ensure at least one line, even for empty cells
         if not wrapped_cell:
@@ -234,17 +261,17 @@ def format_table(rowList):
         # Correct indentation: This should be outside the c_idx loop
         joined_line = f"{BG}{bg_color}{extra}{FG}{Style.Symbol}│{RESET}".join(line_segments)
         # Correct indentation and add missing characters
-        yield f"{Style.MarginSpaces}{joined_line}{RESET}"
+        yield f"{state.space_left()}{FGRESET}{joined_line}{RESET}"
 
     state.bg = BGRESET
 
 def emit_h(level, text):
     text = line_format(text)
-    spaces_to_center = ((state.Width - visible_length(text)) / 2)
+    spaces_to_center = (state.current_width() -  visible_length(text)) / 2
     if level == 1:      #
-        return f"\n{state.space_left()}{BOLD[0]}{' ' * math.floor(spaces_to_center)}{text}{' ' * math.ceil(spaces_to_center)}{BOLD[1]}\n"
+        return f"{state.space_left()}\n{state.space_left()}{BOLD[0]}{' ' * math.floor(spaces_to_center)}{text}{BOLD[1]}"
     elif level == 2:    ##
-        return f"\n{state.space_left()}{BOLD[0]}{FG}{Style.Bright}{' ' * math.floor(spaces_to_center)}{text}{' ' * math.ceil(spaces_to_center)}{RESET}\n\n"
+        return f"{state.space_left()}\n{state.space_left()}{BOLD[0]}{FG}{Style.Bright}{' ' * math.floor(spaces_to_center)}{text}{' ' * math.ceil(spaces_to_center)}{BOLD[1]}{FGRESET}"
     elif level == 3:    ###
         return f"{state.space_left()}{FG}{Style.Head}{BOLD[0]}{text}{RESET}"
     elif level == 4:    ####
@@ -253,13 +280,13 @@ def emit_h(level, text):
         return f"{state.space_left()}{text}{RESET}"
 
 def code_wrap(text_in):
-    if state.WidthWrap and len(text_in) > state.WidthFull:
+    if not Style.PrettyBroken and state.WidthWrap and len(text_in) > state.full_width():
         return (0, [text_in])
 
     # get the indentation of the first line
     indent = len(text_in) - len(text_in.lstrip())
     text = text_in.lstrip()
-    mywidth = state.WidthFull - indent
+    mywidth = state.full_width(-4 if Style.PrettyBroken else 0) - indent
 
     # We take special care to preserve empty lines
     if len(text) == 0:
@@ -277,11 +304,8 @@ def code_wrap(text_in):
 def ansi_collapse(codelist, inp):
     # We break SGR strings into various classes concerning their applicate or removal
     nums = {
-        'fg': r'3\d',
-        'bg': r'4\d',
-        'b': r'2?1',
-        'i': r'2?3',
-        'u': r'2?2',
+        'fg': r'3\d', 'bg': r'4\d',
+        'b': r'2?[12]', 'i': r'2?3', 'u': r'3?2',
         'reset': '0'
     }
 
@@ -311,12 +335,19 @@ def ansi_collapse(codelist, inp):
 
     return codelist + inp
 
-def text_wrap(text, width = -1, indent = 0, first_line_prefix="", subsequent_line_prefix=""):
+
+def split_text(text):
+    return re.split(
+        r'(?<=[\u4E00-\u9FFF\u3400-\u4DBF\uF900-\uFAFF])|(?=[\u4E00-\u9FFF\u3400-\u4DBF\uF900-\uFAFF])|\s+',
+        text
+    )
+
+def text_wrap(text, width = -1, indent = 0, first_line_prefix="", subsequent_line_prefix="", force_truncate=False):
     if width == -1:
         width = state.Width
 
     # The empty word clears the buffer at the end.
-    words = line_format(text).split() + [""]
+    words = split_text(line_format(text)) + [""]
     lines = []
     current_line = ""
     current_style = []
@@ -329,13 +360,21 @@ def text_wrap(text, width = -1, indent = 0, first_line_prefix="", subsequent_lin
             current_style.append(codes.pop(0))
 
         if len(word) and visible_length(current_line) + visible_length(word) + 1 <= width:  # +1 for space
-            current_line += (" " if current_line else "") + word
+
+            current_line += (" " if len(visible(word)) > 0 and current_line and not cjk_count(word) else "") + word
         else:
             # Word doesn't fit, finalize the previous line
             prefix = first_line_prefix if not lines else subsequent_line_prefix
             line_content = prefix + current_line
+            # This is expensive, fix.
+            while force_truncate and visible_length(line_content) >= width:
+                line_content = line_content[:len(line_content) - 2] + "…"
+
             margin = max(0, width - visible_length(line_content))
-            lines.append(line_content + state.bg + ' ' * margin)
+
+            if line_content.strip() != "":
+                lines.append(line_content + state.bg + ' ' * margin)
+
             current_line = (" " * indent) + "".join(current_style) + word
 
         if len(codes):
@@ -349,8 +388,20 @@ def text_wrap(text, width = -1, indent = 0, first_line_prefix="", subsequent_lin
 
     return lines
 
+def cjk_count(s):
+    cjk_re = re.compile(
+        r'[\u4E00-\u9FFF'      # CJK Unified Ideographs
+        r'|\u3400-\u4DBF'       # CJK Unified Ideographs Extension A
+        r'|\uF900-\uFAFF'       # CJK Compatibility Ideographs
+		r'|\uFF00-\uFFEF'       # CJK Compatibility Punctuation
+        r'|\u3000-\u303F'      # CJK Symbols and Punctuation
+        r'|\U0002F800-\U0002FA1F]' # CJK Compatibility Ideographs Supplement
+    )
+    
+    return len(cjk_re.findall(visible(s)))
+
 def line_format(line):
-    not_text = lambda token: not token or len(token.rstrip()) != len(token)
+    not_text = lambda token: not (token.isalnum() or token == '\\') or cjk_count(token)
     footnotes = lambda match: ''.join([chr(SUPER[int(i)]) for i in match.group(1)])
 
     def process_images(match):
@@ -379,7 +430,7 @@ def line_format(line):
     result = ""
 
     for match in tokenList:
-        token = match.group(1)
+        token = re.sub(r'\s+',' ', match.group(1))
         next_token = line[match.end()] if match.end() < len(line) else ""
         prev_token = line[match.start()-1] if match.start() > 0 else ""
 
@@ -435,6 +486,7 @@ def parse(stream):
     last_line_empty_cache = None
     byte = None
     TimeoutIx = 0
+    lexer = None
     while True:
         if state.is_pty or state.is_exec:
             byte = None
@@ -449,7 +501,7 @@ def parse(stream):
                     state.exec_kb += 1
                     os.write(state.exec_master, byte)
 
-                    if byte == b'\n':
+                    if byte in [b'\n', b'\r']:
                         state.buffer = b''
                         print("")
                         state.exec_kb = 0
@@ -484,7 +536,7 @@ def parse(stream):
 
         if not (byte == b'\n' or byte is None): continue
 
-        line = state.buffer.decode('utf-8')
+        line = state.buffer.decode('utf-8').replace('\t','  ')
         state.has_newline = line.endswith('\n')
         # I hate this. There should be better ways.
         state.maybe_prompt = not state.has_newline and state.current()['none'] and re.match(r'^.*>\s+$', visible(line))
@@ -525,7 +577,7 @@ def parse(stream):
                 line = line[len(block_match.group(0)):]
         else:
             if state.block_depth > 0:
-                line = FGRESET + line
+                yield FGRESET
                 state.block_depth = 0
 
         # --- Collapse Multiple Empty Lines if not in code blocks ---
@@ -546,15 +598,16 @@ def parse(stream):
         # \n buffer
         if not state.in_list and len(state.ordered_list_numbers) > 0:
             state.ordered_list_numbers[0] = 0
-        else:
+        elif (not line.startswith(' ' * state.list_indent_text)) and line.strip() != "":
             state.in_list = False
+            state.list_indent_text = 0
 
         if state.first_indent is None:
             state.first_indent = len(line) - len(line.lstrip())
         if len(line) - len(line.lstrip()) >= state.first_indent:
             line = line[state.first_indent:]
         else:
-            logging.warning("Indentation decreased from first line.")
+            logging.debug("Indentation decreased from first line.")
 
 
         # Indent guaranteed
@@ -564,9 +617,7 @@ def parse(stream):
         if state.in_table and not state.in_code and not re.match(r"^\s*\|.+\|\s*$", line):
             state.in_table = False
 
-        #
         # <code><pre>
-        #
         if not state.in_code:
             code_match = re.match(r"^\s*```\s*([^\s]+|$)\s*$", line)
             if code_match:
@@ -581,7 +632,8 @@ def parse(stream):
                     state.code_language = 'Bash'
 
             if state.in_code:
-                state.code_buffer = ""
+                savebrace()
+                state.code_buffer = state.code_buffer_raw = ""
                 state.code_gen = 0
                 state.code_first_line = True
                 state.bg = f"{BG}{Style.Dark}"
@@ -609,7 +661,7 @@ def parse(stream):
                             logging.warning(f"Can't find canonical extension for {state.code_language}")
                             pass
 
-                        open(os.path.join(state.scrape, f"file_{state.scrape_ix}.{ext}"), 'w').write(state.code_buffer)
+                        open(os.path.join(state.scrape, f"file_{state.scrape_ix}.{ext}"), 'w').write(state.code_buffer_raw)
                         state.scrape_ix += 1
 
                     state.code_language = None
@@ -624,7 +676,9 @@ def parse(stream):
 
                     logging.debug(f"code: {state.in_code}")
                     state.emit_flush = True
-                    yield RESET
+                    # We suppress the newline - it's not an explicit style
+                    #state.has_newline = False
+                    #yield RESET
 
                     if code_type == Code.Backtick:
                         continue
@@ -633,7 +687,7 @@ def parse(stream):
                         # nor do we want to be here.
                         raise Goto()
 
-                if state.code_first_line:
+                if state.code_first_line or lexer is None:
                     state.code_first_line = False
                     try:
                         lexer = get_lexer_by_name(state.code_language)
@@ -642,14 +696,16 @@ def parse(stream):
                         lexer = get_lexer_by_name("Bash")
                         custom_style = get_style_by_name("default")
 
-                    formatter = Terminal256Formatter(style=custom_style)
-                    line = line[state.code_indent :]
+                    formatter = TerminalTrueColorFormatter(style=custom_style)
+                    if line.startswith(' ' * state.code_indent):
+                        line = line[state.code_indent :]
 
                 elif line.startswith(" " * state.code_indent):
                     line = line[state.code_indent :]
 
                 # By now we have the properly stripped code line
                 # in the line variable. Add it to the buffer.
+                state.code_buffer_raw += line
                 state.code_line += line
                 if state.code_line.endswith('\n'):
                     line = state.code_line
@@ -660,6 +716,8 @@ def parse(stream):
                 indent, line_wrap = code_wrap(line)
                 
                 state.where_from = "in code"
+                pre = [state.space_left(listwidth = True), '  '] if Style.PrettyBroken else ['', '']
+
                 for tline in line_wrap:
                     # wrap-around is a bunch of tricks. We essentially format longer and longer portions of code. The problem is
                     # the length can change based on look-ahead context so we need to use our expected place (state.code_gen) and
@@ -669,7 +727,7 @@ def parse(stream):
 
                     # Sometimes the highlighter will do things like a full reset or a background reset.
                     # This is not what we want
-                    highlighted_code = re.sub(r"\033\[39(;00|)m", '', highlighted_code)
+                    highlighted_code = re.sub(r"\033\[49(;00|)m", '', highlighted_code)
     
                     # Since we are streaming we ignore the resets and newlines at the end
                     if highlighted_code.endswith(FGRESET + "\n"):
@@ -694,8 +752,8 @@ def parse(stream):
 
                     code_line = ' ' * indent + this_batch.strip()
 
-                    margin = state.WidthFull - visible_length(code_line) % state.WidthFull
-                    yield f"{Style.Codebg}{code_line}{' ' * max(0, margin)}{BGRESET}"  
+                    margin = state.full_width( -len(pre[1]) ) - visible_length(code_line) % state.WidthFull
+                    yield f"{pre[0]}{Style.Codebg}{pre[1]}{code_line}{' ' * max(0, margin)}{BGRESET}"  
                 continue
             except Goto:
                 pass
@@ -705,9 +763,7 @@ def parse(stream):
                 traceback.print_exc()
                 pass
 
-        #
         # <table>
-        #
         if re.match(r"^\s*\|.+\|\s*$", line) and not state.in_code:
             cells = [c.strip() for c in line.strip().strip("|").split("|")]
 
@@ -729,14 +785,28 @@ def parse(stream):
             yield from format_table(cells)
             continue
 
-        #
         # <li> <ul> <ol>
         # llama-4 maverick uses + and +- for lists ... for some reason
-        list_item_match = re.match(r"^(\s*)([\+*\-]|\+\-+|\d+\.)\s+(.*)", line)
+        content = line
+        bullet = ' '
+        list_item_match = re.match(r"^(\s*)([\+*\-] |\+\-+|\d+\.\s+)(.*)", line)
         if list_item_match:
+            # llama 4 maverick does this weird output like this
+            # 1. blah blah blah
+            #    this should be a list
+            #    
+            #    ```bash
+            #    blah blah
+            #    ```
+            #
+            #    still in the list
+            # We do this here so that the first line which is the bullet
+            # line gets the proper hang
+            state.list_indent_text = len(list_item_match.group(2)) - 1
             state.in_list = True
 
             indent = len(list_item_match.group(1))
+
             list_type = "number" if list_item_match.group(2)[0].isdigit() else "bullet"
             content = list_item_match.group(3)
 
@@ -756,34 +826,34 @@ def parse(stream):
             if list_type == "number":
                 state.ordered_list_numbers[-1] += 1
 
-            indent = (len(state.list_item_stack) - 1) * 2
-
-            wrap_width = state.Width - indent - (2 * Style.ListIndent) 
-
             bullet = '•'
             if list_type == "number":
                 list_number = int(max(state.ordered_list_numbers[-1], float(list_item_match.group(2))))
                 bullet = str(list_number)
+
+        # This is intentional ... we can get here in llama 4 using
+        # a weird thing
+        if state.in_list:
+            indent = (len(state.list_item_stack) - 1) * Style.ListIndent + (len(bullet) - 1)
+            wrap_width = state.current_width() - indent - (2 * Style.ListIndent) 
             
             wrapped_lineList = text_wrap(content, wrap_width, Style.ListIndent,
-                first_line_prefix      = f"{(' ' * (indent ))}{FG}{Style.Symbol}{bullet}{RESET} ",
+                first_line_prefix = f"{(' ' * indent)}{FG}{Style.Symbol}{bullet}{RESET} ",
                 subsequent_line_prefix = " " * (indent)
             )
             for wrapped_line in wrapped_lineList:
                 yield f"{state.space_left()}{wrapped_line}\n"
+
             continue
-        #
+
         # <h1> ... <h6>
-        # 
         header_match = re.match(r"^\s*(#{1,6})\s+(.*)", line)
         if header_match:
             level = len(header_match.group(1))
             yield emit_h(level, header_match.group(2))
             continue
 
-        #
         # <hr>
-        #
         hr_match = re.match(r"^[\s]*([-\*=_]){3,}[\s]*$", line)
         if hr_match:
             if state.last_line_empty or last_line_empty_cache:
@@ -800,7 +870,7 @@ def parse(stream):
         if len(line) == 0: yield ""
         if len(line) < state.Width:
             # we want to prevent word wrap
-            yield f"{state.space_left()}{line_format(line)}"
+            yield f"{state.space_left()}{line_format(line.lstrip())}"
         else:
             wrapped_lines = text_wrap(line)
             for wrapped_line in wrapped_lines:
@@ -845,10 +915,10 @@ def emit(inp):
         else:
             chunk = buffer.pop(0)
 
-        print(chunk, end="", flush=True)
+        print(chunk, end="", file=sys.stdout, flush=True)
 
     if len(buffer):
-        print(buffer.pop(0), end="", flush=True)
+        print(buffer.pop(0), file=sys.stdout, end="", flush=True)
 
 def apply_multipliers(name, H, S, V):
     m = _style.get(name)
@@ -871,15 +941,16 @@ def width_calc():
             state.WidthFull = width
 
     state.Width = state.WidthFull - 2 * Style.Margin
+    pre = state.space_left(listwidth=True) if Style.PrettyBroken else ''
     Style.Codepad = [
-        f"{RESET}{FG}{Style.Dark}{'▄' * state.WidthFull}{RESET}\n",
-        f"{RESET}{FG}{Style.Dark}{'▀' * state.WidthFull}{RESET}"
+        f"{pre}{RESET}{FG}{Style.Dark}{'▄' * state.full_width()}{RESET}\n",
+        f"{pre}{RESET}{FG}{Style.Dark}{'▀' * state.full_width()}{RESET}"
     ]
 
 def main():
     global H, S, V
 
-    parser = ArgumentParser(description="Streamdown - A markdown renderer for modern terminals")
+    parser = ArgumentParser(description="Streamdown - A Streaming markdown renderer for modern terminals")
     parser.add_argument("filenameList", nargs="*", help="Input file to process (also takes stdin)")
     parser.add_argument("-l", "--loglevel", default="INFO", help="Set the logging level")
     parser.add_argument("-c", "--color", default=None, help="Set the hsv base: h,s,v")
@@ -896,20 +967,20 @@ def main():
 
     for color in ["Dark", "Mid", "Symbol", "Head", "Grey", "Bright"]:
         setattr(Style, color, apply_multipliers(color, H, S, V))
-    for attr in ['Margin', 'ListIndent', 'Syntax']:
+    for attr in ['PrettyBroken', 'Margin', 'ListIndent', 'Syntax']:
         setattr(Style, attr, _style.get(attr))
-    
+
     if args.scrape:
         os.makedirs(args.scrape, exist_ok=True)
         state.scrape = args.scrape
 
     Style.MarginSpaces = " " * Style.Margin
     state.WidthArg = int(args.width) or _style.get("Width") or 0
+    Style.Blockquote = f"{FG}{Style.Grey}│ "
     width_calc()
 
     Style.Codebg = f"{BG}{Style.Dark}"
     Style.Link = f"{FG}{Style.Symbol}{UNDERLINE[0]}"
-    Style.Blockquote = f"{FG}{Style.Grey}│ "
 
     logging.basicConfig(stream=sys.stdout, level=args.loglevel.upper(), format=f'%(message)s')
     state.exec_master, state.exec_slave = pty.openpty()
@@ -922,6 +993,7 @@ def main():
             os.close(state.exec_slave)  # We don't need slave in parent
             # Set stdin to raw mode so we don't need to press enter
             tty.setcbreak(sys.stdin.fileno())
+            sys.stdout.write("\x1b[?7h")
             emit(sys.stdin)
 
         elif args.filenameList:
@@ -950,15 +1022,14 @@ def main():
         logging.warning(f"Exception thrown: {type(ex)} {ex}")
         traceback.print_exc()
 
-    if state.Clipboard and state.code_buffer:
-        code = state.code_buffer
+    if state.Clipboard and state.code_buffer_raw:
+        code = state.code_buffer_raw
         # code needs to be a base64 encoded string before emitting
         code_bytes = code.encode('utf-8')
         base64_bytes = base64.b64encode(code_bytes)
         base64_string = base64_bytes.decode('utf-8')
         print(f"\033]52;c;{base64_string}\a", end="", flush=True)
 
-
     if state.terminal:
         termios.tcsetattr(sys.stdin, termios.TCSADRAIN, state.terminal)
         os.close(state.exec_master)

streamdown/ss

@@ -0,0 +1 @@
+*   **Model Card:**  Always read the model card on the Hugging Face Hub ([https://huggingface.co/microsoft/bitnet-b1.58-2B-4T](https://huggingface.co/microsoft/bitnet-b1.58-2B-4T)) for important information about the model, its intended use, limitations, and potential biases.

streamdown/ss1

@@ -0,0 +1,42 @@
+    *   `model.safetensors`: The name of the file you want to download.  You'll need to know the exact filename.  You can find the files in the model repository on the Hugging Face Hub website ([https://huggingface.co/microsoft/bitnet-b1.58-2B-4T](https://huggingface.co/microsoft/bitnet-b1.58-2B-4T)).  Look under the "Files and versions" tab.  `safetensors` is the preferred format for model weights now.  If it's a `.bin` file, you can download that instead.
+    *   `--local-dir ./bitnet-b1.58-2B-4T`:  The directory to save the file to.
+
+*   **Download using `transformers` library (recommended for most use cases):**
+
+    The `transformers` library provides a convenient way to download and cache models.  This is often the easiest approach if you're using the model with `transformers`.  You don't *directly* use the `huggingface-cli` for this, but it's worth knowing.
+
+    ```python
+    from transformers import AutoModelForCausalLM, AutoTokenizer
+
+    model_name = "microsoft/bitnet-b1.58-2B-4T"
+
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModelForCausalLM.from_pretrained(model_name)
+
+    # The model and tokenizer will be downloaded and cached in your
+    # transformers cache directory (usually ~/.cache/huggingface/transformers).
+    ```
+
+    This approach automatically handles downloading the necessary files and caching them for future use.  It also handles the correct file formats and configurations.
+
+**4.  Checking the Download**
+
+After the download completes, verify that the files are in the specified directory.  You can use `ls` (Linux/macOS) or `dir` (Windows) to list the contents of the directory.
+
+**Important Considerations:**
+
+*   **Disk Space:**  The `bitnet-b1.58-2B-4T` model is quite large (several gigabytes).  Make sure you have enough free disk space before downloading.
+*   **Network Connection:**  A stable and fast internet connection is essential for a smooth download.
+*   **Caching:** The Hugging Face Hub and `transformers` library use caching to avoid re-downloading models unnecessarily.  The default cache directory is usually `~/.cache/huggingface/transformers`.
+*   **File Formats:**  Models are often stored in `safetensors` or `.bin` formats.  `safetensors` is generally preferred for security and performance.
+*   **Model Card:**  Always read the model card on the Hugging Face Hub ([https://huggingface.co/microsoft/bitnet-b1.58-2B-4T](https://huggingface.co/microsoft/bitnet-b1.58-2B-4T)) for important information about the model, its intended use, limitations, and potential biases.
+*   **Gated Models:** Some models require you to accept terms of use before you can download them.  The `huggingface-cli login` command will guide you through this process if necessary.
+
+**Example Workflow (Recommended):**
+
+1.  `huggingface-cli login` (if not already logged in)
+2.  Use the `transformers` library in a Python script to download and load the model (as shown in the example above).  This is the most convenient and reliable method for most use cases.
+
+Let me know if you have any other questions or if you'd like help with a specific task related to this model!
+
+> 

{streamdown-0.16.0.dist-info → streamdown-0.18.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: streamdown
-Version: 0.16.0
+Version: 0.18.0
 Summary: A streaming markdown renderer for modern terminals with syntax highlighting
 Project-URL: Homepage, https://github.com/kristopolous/Streamdown
 Project-URL: Bug Tracker, https://github.com/kristopolous/Streamdown/issues
@@ -31,18 +31,22 @@ Description-Content-Type: text/markdown
 <img src=https://github.com/user-attachments/assets/0468eac0-2a00-4e98-82ca-09e6ac679357/>
 <br/>
 <a href=https://pypi.org/project/streamdown><img src=https://badge.fury.io/py/streamdown.svg/></a>
+<br/><strong>Terminal streaming markdown that rocks</strong>
+
 </p>
 
-The streaming markdown renderer for the terminal that rocks!
-Streamdown works with [simonw's llm](https://github.com/simonw/llm) along with any other streaming markdown. You even get full readline and keyboard navigation support.
+
+Streamdown works with [simonw's llm](https://github.com/simonw/llm) along with any other streaming markdown, even something basic like curl. 
+It supports standard piping like any normal pager and a clean `execvp` option for robustly wrapping around interactive programs with readline or their own ANSI stuff to manage.
 ```bash
 $ pip install streamdown
 ```
 ![Streamdown is Amazing](https://github.com/user-attachments/assets/268cb340-78cc-4df0-a773-c5ac95eceeeb)
 
 ### Provides clean copyable code for long code lines
-You may have noticed *inferior* renderers inject line breaks when copying code that wraps around. We're better and now you are too!
+Some *inferior* renderers inject line breaks when copying code that wraps around. We're better and now you are too!
 ![Handle That Mandle](https://github.com/user-attachments/assets/a27aa70c-f691-4796-84f0-c2eb18c7de23)
+**Tip**: You can make things prettier if you don't mind if this guarantee is broken. See the `PrettyBroken` flag below!
 
 ### Supports images
 Here's kitty and alacritty. Try to do that in glow...
@@ -57,6 +61,9 @@ Here's kitty and alacritty. Try to do that in glow...
 As well as everything else...
 ![dunder](https://github.com/user-attachments/assets/d41d7fec-6dec-4387-b53d-f2098f269a5e)
 
+Very ... Carefully ... Supported ... 
+![cjk1](https://github.com/user-attachments/assets/75162ade-4734-440e-aaa3-5ffc17a0dd46)
+
 ### Colors are highly (and quickly) configurable for people who care a lot, or just a little.
 ![configurable](https://github.com/user-attachments/assets/19ca2ec9-8ea1-4a79-87ca-8352789269fe)
 
@@ -67,7 +74,7 @@ For instance, here is the [latex plugin](https://github.com/kristopolous/Streamd
 
 ## TOML Configuration 
 
-Streamdown uses a TOML configuration file located at `~/.config/streamdown/config.toml` (following the XDG Base Directory Specification). If this file does not exist upon first run, it will be created with default values. 
+It's located at `~/.config/streamdown/config.toml` (following the XDG Base Directory Specification). If this file does not exist upon first run, it will be created with default values. 
 
 Here are the sections:
 
@@ -85,12 +92,15 @@ Defines the base Hue (H), Saturation (S), and Value (V) from which all other pal
 *   `Margin` (integer, default: `2`): The left and right indent for the output. 
 *   `Width` (integer, default: `0`): Along with the `Margin`, `Width` specifies the base width of the content, which when set to 0, means use the terminal width. See [#6](https://github.com/kristopolous/Streamdown/issues/6) for more details
 *   `PrettyPad` (boolean, default: `false`): Uses a unicode vertical pad trick to add a half height background to code blocks. This makes copy/paste have artifacts. See [#2](https://github.com/kristopolous/Streamdown/issues/2). I like it on. But that's just me
+*   `PrettyBroken` (boolean, default: `false`): This will break the copy/paste assurance above. The output is much prettier, but it's also broken. So it's pretty broken. Works nicely with PrettyPad.
 *   `ListIndent` (integer, default: `2`): This is the recursive indent for the list styles.
 *   `Syntax` (string, default `monokai`): This is the syntax [highlighting theme which come via pygments](https://pygments.org/styles/).
 
 Example:
 ```toml
 [style]
+PrettyPad = true
+PrettyBroken = true
 HSV = [0.7, 0.5, 0.5]
 Dark = { H = 1.0, S = 1.2, V = 0.25 } # Make dark elements less saturated and darker
 Symbol = { H = 1.0, S = 1.8, V = 1.8 } # Make symbols more vibrant
@@ -103,16 +113,13 @@ Controls optional features:
 *   `CodeSpaces` (boolean, default: `true`): Enables detection of code blocks indented with 4 spaces. Set to `false` to disable this detection method (triple-backtick blocks still work).
 *   `Clipboard` (boolean, default: `true`): Enables copying the last code block encountered to the system clipboard using OSC 52 escape sequences upon exit. Set to `false` to disable.
 *   `Logging` (boolean, default: `false`): Enables logging to tmpdir (/tmp/sd) of the raw markdown for debugging and bug reporting. The logging uses an emoji as a record separator so the actual streaming delays can be simulated and replayed. If you use the `filename` based invocation, that is to say, `sd <filename>`, this type of logging is always off.
-*   `Timeout` (float, default: `0.5`): This is a workaround to the [buffer parsing bugs](https://github.com/kristopolous/Streamdown/issues/4). By increasing the select timeout, the parser loop only gets triggerd on newline which means that having to resume from things like a code block, inside a list, inside a table, between buffers, without breaking formatting doesn't need to be done. I assert (2025-04-09) this is no longer a bug. Feel free to turn on `Logging` and post an issue if you find a repeatable one. 
+*   `Savebrace` (boolean, default: `true`): Saves the code blocks of a conversation to the append file `/tmp/sd/savebrace` so you can fzf or whatever you want through it. See how it's used in my [llmehelp](https://github.com/kristopolous/llmehelp) scripts, specifically `screen-query` and `sd-picker`.
 
 Example:
 ```toml
 [features]
 CodeSpaces = false
 Clipboard = false
-Margin = 4
-Width = 120
-Timeout = 1.0
 ```
 
 ## Command Line
@@ -146,7 +153,7 @@ Do this
     $ ./streamdown/sd.py tests/*md
 
 ## Install from source
-After the git clone least one of these should work, hopefully. it's using the modern uv pip tool.
+After the git clone least one of these should work, hopefully. it's using the modern uv pip tool but is also backwards compatible to the `pip3 install -r requirements.txt` flow.
 
     $ pipx install -e .
     $ pip install -e .
@@ -154,9 +161,5 @@ After the git clone least one of these should work, hopefully. it's using the mo
 
 ### Future work
 
-#### CSS
-I'm really considering using `tinycss2` and making an actual stylesheet engine. This is related to another problem - getting a modern HTML renderer in the terminal that is actually navigable. I *think* it's probably a separate project.
-
-#### scrape
-This is already partially implemented. The idea is every code block can get extracted and put in a directory so you can have a conversation to generate every piece of a project, similar to Aider, Claude or Goose,  but in the most hands-off yet still convenient way possible.
-
+#### Glow styles
+I'm going to try to be compatible with other popular markdown styles to help for a smoother transition. Glow compatible json sheets is on my radar. There's also mdless and frogmouth. Might be others

streamdown-0.18.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+streamdown/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+streamdown/sd.py,sha256=gFm6WqrWsMqV8EW9tcq7ebGoXpDFrTd89_ka21AaHm8,39118
+streamdown/ss,sha256=sel_phpaecrw6WGIHRLROsD7BFShf0rSDHheflwdUn8,277
+streamdown/ss1,sha256=CUVf86_2zeAle2oQCeTfWYqtHBrAFR_UgvptuYMQzFU,3151
+streamdown/plugins/README.md,sha256=KWqYELs9WkKJmuDzYv3cvPlZMkArsNCBUe4XDoTLjLA,1143
+streamdown/plugins/latex.py,sha256=xZMGMdx_Sw4X1piZejXFHfEG9qazU4fGeceiMI0h13Y,648
+streamdown-0.18.0.dist-info/METADATA,sha256=Lyrf0k6BjC4wjiwwWz5b7aOFPtLR5uJ41e-SRGr1JC0,8062
+streamdown-0.18.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+streamdown-0.18.0.dist-info/entry_points.txt,sha256=HroKFsFMGf_h9PRTE96NjvjJQWupMW5TGP5RGUr1O_Q,74
+streamdown-0.18.0.dist-info/licenses/LICENSE.MIT,sha256=SnY46EPirUsF20dZDR8HpyVgS2_4Tjxuc6f-4OdqO7U,1070
+streamdown-0.18.0.dist-info/RECORD,,

streamdown/scrape/file_0.py

@@ -1,22 +0,0 @@
-def fizzbuzz(n):
-for i in range(1, n + 1):
-if i % 3 == 0 and i % 5 == 0:
-print("FizzBuzz")
-elif i % 3 == 0:
-print("Fizz")
-elif i % 5 == 0:
-print("Buzz")
-else:
-print(i)
-
-# Example usage: Print FizzBuzz up to 100 Example usage: Print FizzBuzz up to 100 Example usage: Print FizzBuzz up to 100 Example usage: Print FizzBuzz up to 100
-fizzbuzz(100)
-
-#  Example usage: different range:
-fizzbuzz(20)
-
-#Example usage:  one line output (list comprehension)
-def fizzbuzz_oneline(n):
-print(["FizzBuzz" if i%3==0 and i%5==0 else "Fizz" if i%3==0 else "Buzz" if i%5==0 else i for i in range(1,n+1)])
-
-fizzbuzz_oneline(30)

streamdown/scrape/file_1.js

@@ -1,27 +0,0 @@
-function fizzBuzz(n) {
-for (let i = 1; i <= n; i++) {
-if (i % 3 === 0 && i % 5 === 0) {
-console.log("FizzBuzz");
-} else if (i % 3 === 0) {
-console.log("Fizz");
-} else if (i % 5 === 0) {
-console.log("Buzz");
-} else {
-console.log(i);
-}
-}
-}
-
-// Example usage:
-fizzBuzz(100);
-
-// Example usage: different range
-fizzBuzz(25);
-
-// Example one-line output. (arrow function & ternary operator)
-const fizzBuzzOneLine = n => {
-for (let i = 1; i <= n; i++) {
-console.log((i % 3 === 0 ? (i % 5 === 0 ? "FizzBuzz" : "Fizz") : (i % 5 === 0 ? "Buzz" : i)));
-}
-};
-fizzBuzzOneLine(30);

streamdown/scrape/file_2.cpp

@@ -1,23 +0,0 @@
-#include <iostream>
-
-void fizzBuzz(int n) {
-for (int i = 1; i <= n; i++) {
-if (i % 3 == 0 && i % 5 == 0) {
-std::cout << "FizzBuzz" << std::endl;
-} else if (i % 3 == 0) {
-std::cout << "Fizz" << std::endl;
-} else if (i % 5 == 0) {
-std::cout << "Buzz" << std::endl;
-} else {
-std::cout << i << std::endl;
-}
-}
-}
-
-int main() {
-fizzBuzz(100);
-
-// Example usage: different range
-fizzBuzz(35);
-return 0;
-}

streamdown/tt.mds

@@ -1,11 +0,0 @@
-                 **A markdown renderer for modern terminals**
-                 ##### Usage examples:
-
-                 ``` bash
-                 sd [filename]
-                 cat README.md | sd
-                 stdbuf -oL llm chat | sd
-                 ```
-
-                 If no filename is provided and no input is piped, this help message is displayed.
-

streamdown-0.16.0.dist-info/RECORD

@@ -1,13 +0,0 @@
-streamdown/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-streamdown/sd.py,sha256=0Ug2grAsf_RWVMyDMOc0j_ZTmL8N6dOPpk24LKBPkhA,35641
-streamdown/tt.mds,sha256=srDldQ9KnMJd5P8GdTXTJl4mjTowwV9y58ZIaBVbtFY,359
-streamdown/plugins/README.md,sha256=KWqYELs9WkKJmuDzYv3cvPlZMkArsNCBUe4XDoTLjLA,1143
-streamdown/plugins/latex.py,sha256=xZMGMdx_Sw4X1piZejXFHfEG9qazU4fGeceiMI0h13Y,648
-streamdown/scrape/file_0.py,sha256=OiFxFGGHu2C2iO9LVnhXKCybqCsnw0bu8MmI2E0vs_s,610
-streamdown/scrape/file_1.js,sha256=JnXSvlsk9UmU5LsGOfXkP3sGId8VNEJRJo8-uRohRCM,569
-streamdown/scrape/file_2.cpp,sha256=4hbT9TJzDNmrU7BVwaIuCMlI2BvUEVeTKoH6wUJRkrI,397
-streamdown-0.16.0.dist-info/METADATA,sha256=sq6eIyimqRI8cZHKkNbDdP3OoYpEPYCU-v5LFhBgYfQ,7968
-streamdown-0.16.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-streamdown-0.16.0.dist-info/entry_points.txt,sha256=HroKFsFMGf_h9PRTE96NjvjJQWupMW5TGP5RGUr1O_Q,74
-streamdown-0.16.0.dist-info/licenses/LICENSE.MIT,sha256=SnY46EPirUsF20dZDR8HpyVgS2_4Tjxuc6f-4OdqO7U,1070
-streamdown-0.16.0.dist-info/RECORD,,