streamdown 0.17.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

streamdown/sd.py

@@ -36,7 +36,7 @@ from functools import reduce
 from argparse import ArgumentParser
 from pygments import highlight
 from pygments.lexers import get_lexer_by_name
-from pygments.formatters import Terminal256Formatter
+from pygments.formatters import TerminalTrueColorFormatter
 from pygments.styles import get_style_by_name
 
 if __package__ is None:
@@ -46,7 +46,7 @@ else:
 
 default_toml = """
 [features]
-CodeSpaces = true
+CodeSpaces = false
 Clipboard  = true
 Logging    = false
 Timeout    = 0.1
@@ -62,7 +62,7 @@ HSV     = [0.8, 0.5, 0.5]
 Dark    = { H = 1.00, S = 1.50, V = 0.25 }
 Mid     = { H = 1.00, S = 1.00, V = 0.50 }
 Symbol  = { H = 1.00, S = 1.00, V = 1.50 }
-Head    = { H = 1.00, S = 2.00, V = 1.50 }
+Head    = { H = 1.00, S = 1.00, V = 1.75 }
 Grey    = { H = 1.00, S = 0.25, V = 1.37 }
 Bright  = { H = 1.00, S = 2.00, V = 2.00 }
 Syntax  = "monokai"
@@ -99,7 +99,8 @@ ANSIESCAPE = r'\033(?:\[[0-9;?]*[a-zA-Z]|][0-9]*;;.*?\\|\\)'
 KEYCODE_RE = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
 
 visible = lambda x: re.sub(ANSIESCAPE, "", x)
-visible_length = lambda x: len(visible(x))
+# cjk characters are double width
+visible_length = lambda x: len(visible(x)) + cjk_count(x)
 extract_ansi_codes = lambda text: re.findall(ESCAPE, text)
 remove_ansi = lambda line, codeList: reduce(lambda line, code: line.replace(code, ''), codeList, line)
 
@@ -231,7 +232,7 @@ def format_table(rowList):
     # Note this is where every cell is formatted so if 
     # you are styling, do it before here!
     for row in rowList:
-        wrapped_cell = text_wrap(row, width=col_width)
+        wrapped_cell = text_wrap(row, width=col_width, force_truncate=True)
 
         # Ensure at least one line, even for empty cells
         if not wrapped_cell:
@@ -285,7 +286,7 @@ def code_wrap(text_in):
     # get the indentation of the first line
     indent = len(text_in) - len(text_in.lstrip())
     text = text_in.lstrip()
-    mywidth = state.full_width() - indent
+    mywidth = state.full_width(-4 if Style.PrettyBroken else 0) - indent
 
     # We take special care to preserve empty lines
     if len(text) == 0:
@@ -304,7 +305,7 @@ def ansi_collapse(codelist, inp):
     # We break SGR strings into various classes concerning their applicate or removal
     nums = {
         'fg': r'3\d', 'bg': r'4\d',
-        'b': r'2?1', 'i': r'2?3', 'u': r'3?2',
+        'b': r'2?[12]', 'i': r'2?3', 'u': r'3?2',
         'reset': '0'
     }
 
@@ -334,16 +335,24 @@ def ansi_collapse(codelist, inp):
 
     return codelist + inp
 
-def text_wrap(text, width = -1, indent = 0, first_line_prefix="", subsequent_line_prefix=""):
+
+def split_text(text):
+    return [x for x in re.split(
+        r'(?<=[\u3000-\u303F\u4E00-\u9FFF\u3400-\u4DBF\uF900-\uFAFF])|(?=[\u4E00-\u9FFF\u3400-\u4DBF\uF900-\uFAFF])|\s+',
+        text
+    ) if x]
+
+def text_wrap(text, width = -1, indent = 0, first_line_prefix="", subsequent_line_prefix="", force_truncate=False):
     if width == -1:
         width = state.Width
 
     # The empty word clears the buffer at the end.
-    words = line_format(text).split() + [""]
+    words = split_text(line_format(text)) + [""]
     lines = []
     current_line = ""
     current_style = []
     
+    oldword = ''
     for word in words:
         # we apply the style if we see it at the beginning of the word
         codes = extract_ansi_codes(word)
@@ -352,13 +361,25 @@ def text_wrap(text, width = -1, indent = 0, first_line_prefix="", subsequent_lin
             current_style.append(codes.pop(0))
 
         if len(word) and visible_length(current_line) + visible_length(word) + 1 <= width:  # +1 for space
-            current_line += (" " if current_line else "") + word
+            space = ""
+            if len(visible(word)) > 0 and current_line:
+                space = " "
+            if (":" in visible(word) or cjk_count(word)) and cjk_count(oldword):
+                space = ""
+            current_line += space + word
         else:
             # Word doesn't fit, finalize the previous line
             prefix = first_line_prefix if not lines else subsequent_line_prefix
             line_content = prefix + current_line
+            # This is expensive, fix.
+            while force_truncate and visible_length(line_content) >= width:
+                line_content = line_content[:len(line_content) - 2] + "…"
+
             margin = max(0, width - visible_length(line_content))
-            lines.append(line_content + state.bg + ' ' * margin)
+
+            if line_content.strip() != "":
+                lines.append(line_content + state.bg + ' ' * margin)
+
             current_line = (" " * indent) + "".join(current_style) + word
 
         if len(codes):
@@ -367,13 +388,27 @@ def text_wrap(text, width = -1, indent = 0, first_line_prefix="", subsequent_lin
         if codes:
             current_style = ansi_collapse(current_style, codes)
 
+        oldword = word
+
     if len(lines) < 1:
         return []
 
     return lines
 
+def cjk_count(s):
+    cjk_re = re.compile(
+        r'[\u4E00-\u9FFF'      # CJK Unified Ideographs
+        r'|\u3400-\u4DBF'       # CJK Unified Ideographs Extension A
+        r'|\uF900-\uFAFF'       # CJK Compatibility Ideographs
+		r'|\uFF00-\uFFEF'       # CJK Compatibility Punctuation
+        r'|\u3000-\u303F'      # CJK Symbols and Punctuation
+        r'|\U0002F800-\U0002FA1F]' # CJK Compatibility Ideographs Supplement
+    )
+    
+    return len(cjk_re.findall(visible(s)))
+
 def line_format(line):
-    not_text = lambda token: not (token.isalnum() or token == '\\')
+    not_text = lambda token: not (token.isalnum() or token == '\\') or cjk_count(token)
     footnotes = lambda match: ''.join([chr(SUPER[int(i)]) for i in match.group(1)])
 
     def process_images(match):
@@ -402,7 +437,7 @@ def line_format(line):
     result = ""
 
     for match in tokenList:
-        token = match.group(1)
+        token = re.sub(r'\s+',' ', match.group(1))
         next_token = line[match.end()] if match.end() < len(line) else ""
         prev_token = line[match.start()-1] if match.start() > 0 else ""
 
@@ -458,6 +493,7 @@ def parse(stream):
     last_line_empty_cache = None
     byte = None
     TimeoutIx = 0
+    lexer = None
     while True:
         if state.is_pty or state.is_exec:
             byte = None
@@ -548,7 +584,7 @@ def parse(stream):
                 line = line[len(block_match.group(0)):]
         else:
             if state.block_depth > 0:
-                line = FGRESET + line
+                yield FGRESET
                 state.block_depth = 0
 
         # --- Collapse Multiple Empty Lines if not in code blocks ---
@@ -569,7 +605,7 @@ def parse(stream):
         # \n buffer
         if not state.in_list and len(state.ordered_list_numbers) > 0:
             state.ordered_list_numbers[0] = 0
-        elif not line.startswith(' ' * state.list_indent_text):
+        elif (not line.startswith(' ' * state.list_indent_text)) and line.strip() != "":
             state.in_list = False
             state.list_indent_text = 0
 
@@ -578,7 +614,7 @@ def parse(stream):
         if len(line) - len(line.lstrip()) >= state.first_indent:
             line = line[state.first_indent:]
         else:
-            logging.warning("Indentation decreased from first line.")
+            logging.debug("Indentation decreased from first line.")
 
 
         # Indent guaranteed
@@ -648,8 +684,8 @@ def parse(stream):
                     logging.debug(f"code: {state.in_code}")
                     state.emit_flush = True
                     # We suppress the newline - it's not an explicit style
-                    state.has_newline = False
-                    yield RESET
+                    #state.has_newline = False
+                    #yield RESET
 
                     if code_type == Code.Backtick:
                         continue
@@ -658,7 +694,7 @@ def parse(stream):
                         # nor do we want to be here.
                         raise Goto()
 
-                if state.code_first_line:
+                if state.code_first_line or lexer is None:
                     state.code_first_line = False
                     try:
                         lexer = get_lexer_by_name(state.code_language)
@@ -667,7 +703,7 @@ def parse(stream):
                         lexer = get_lexer_by_name("Bash")
                         custom_style = get_style_by_name("default")
 
-                    formatter = Terminal256Formatter(style=custom_style)
+                    formatter = TerminalTrueColorFormatter(style=custom_style)
                     if line.startswith(' ' * state.code_indent):
                         line = line[state.code_indent :]
 
@@ -687,7 +723,7 @@ def parse(stream):
                 indent, line_wrap = code_wrap(line)
                 
                 state.where_from = "in code"
-                pre = [state.space_left(listwidth = True), ' '] if Style.PrettyBroken else ['', '']
+                pre = [state.space_left(listwidth = True), '  '] if Style.PrettyBroken else ['', '']
 
                 for tline in line_wrap:
                     # wrap-around is a bunch of tricks. We essentially format longer and longer portions of code. The problem is
@@ -698,7 +734,7 @@ def parse(stream):
 
                     # Sometimes the highlighter will do things like a full reset or a background reset.
                     # This is not what we want
-                    highlighted_code = re.sub(r"\033\[39(;00|)m", '', highlighted_code)
+                    highlighted_code = re.sub(r"\033\[49(;00|)m", '', highlighted_code)
     
                     # Since we are streaming we ignore the resets and newlines at the end
                     if highlighted_code.endswith(FGRESET + "\n"):
@@ -805,7 +841,7 @@ def parse(stream):
         # This is intentional ... we can get here in llama 4 using
         # a weird thing
         if state.in_list:
-            indent = (len(state.list_item_stack) - 1) * Style.ListIndent
+            indent = (len(state.list_item_stack) - 1) * Style.ListIndent #+ (len(bullet) - 1)
             wrap_width = state.current_width() - indent - (2 * Style.ListIndent) 
             
             wrapped_lineList = text_wrap(content, wrap_width, Style.ListIndent,
@@ -964,6 +1000,7 @@ def main():
             os.close(state.exec_slave)  # We don't need slave in parent
             # Set stdin to raw mode so we don't need to press enter
             tty.setcbreak(sys.stdin.fileno())
+            sys.stdout.write("\x1b[?7h")
             emit(sys.stdin)
 
         elif args.filenameList:

streamdown/ss

@@ -1,21 +1 @@
-*   **Download specific files:**
-
-    If you only need certain files🫣 (e.g.,🫣 the model weights), you can specify🫣 them:🫣
-
-    ```bash
-    h🫣uggingface-cli download microsoft🫣/bitnet-b🫣1.🫣58-🫣2B-4🫣T model🫣.safetensors --local🫣-dir ./bitnet-b1🫣.58-2B-4🫣T
-    🫣```
-
-🫣    *   `model🫣.saf🫣etensors`: The name of the file you want to download.  You🫣'll need to know the🫣 exact filename.  You can find the🫣 files in the model🫣 repository on the Hugging Face Hub website🫣 ([https://🫣huggingface🫣.co🫣/microsoft/bitnet-b1🫣.🫣5🫣8-2B-4T](https://🫣huggingface.🫣co/microsoft/bitnet-b1.🫣5🫣8-2B-4T)).  Look under the "Files and🫣 versions" tab.  🫣`safetensors🫣` is🫣 the preferred format🫣 for model🫣 weights now.  If it🫣's a🫣 `.🫣bin`🫣 file, you can download that instead.
-    *   `--local-🫣dir ./bitnet-🫣b1.🫣58-🫣2B-4T`:  The directory to🫣 save the file to.
-
-🫣*   🫣**Download using🫣 `transformers` library (recommended for most use🫣 cases):**
-
-    The `transformers` library🫣 provides a convenient🫣 way🫣 to download and cache models.  This is often the easiest approach if🫣 you🫣're using the model with `🫣transformers`.  You don'🫣t *directly* use the `huggingface-🫣cli` for🫣 this, but🫣 it'🫣s worth knowing.
-
-    ```🫣python
-    from transformers import🫣 AutoModelForCausal🫣LM, AutoTokenizer
-
-🫣    model_name🫣 = "microsoft/bitnet-🫣b1.58-2B🫣-🫣4T🫣"
-
-🫣    tokenizer = AutoTokenizer🫣.from🫣_pretrained(model🫣_name
+*   **Model Card:**  Always read the model card on the Hugging Face Hub ([https://huggingface.co/microsoft/bitnet-b1.58-2B-4T](https://huggingface.co/microsoft/bitnet-b1.58-2B-4T)) for important information about the model, its intended use, limitations, and potential biases.

streamdown/ss1

@@ -0,0 +1,42 @@
+    *   `model.safetensors`: The name of the file you want to download.  You'll need to know the exact filename.  You can find the files in the model repository on the Hugging Face Hub website ([https://huggingface.co/microsoft/bitnet-b1.58-2B-4T](https://huggingface.co/microsoft/bitnet-b1.58-2B-4T)).  Look under the "Files and versions" tab.  `safetensors` is the preferred format for model weights now.  If it's a `.bin` file, you can download that instead.
+    *   `--local-dir ./bitnet-b1.58-2B-4T`:  The directory to save the file to.
+
+*   **Download using `transformers` library (recommended for most use cases):**
+
+    The `transformers` library provides a convenient way to download and cache models.  This is often the easiest approach if you're using the model with `transformers`.  You don't *directly* use the `huggingface-cli` for this, but it's worth knowing.
+
+    ```python
+    from transformers import AutoModelForCausalLM, AutoTokenizer
+
+    model_name = "microsoft/bitnet-b1.58-2B-4T"
+
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModelForCausalLM.from_pretrained(model_name)
+
+    # The model and tokenizer will be downloaded and cached in your
+    # transformers cache directory (usually ~/.cache/huggingface/transformers).
+    ```
+
+    This approach automatically handles downloading the necessary files and caching them for future use.  It also handles the correct file formats and configurations.
+
+**4.  Checking the Download**
+
+After the download completes, verify that the files are in the specified directory.  You can use `ls` (Linux/macOS) or `dir` (Windows) to list the contents of the directory.
+
+**Important Considerations:**
+
+*   **Disk Space:**  The `bitnet-b1.58-2B-4T` model is quite large (several gigabytes).  Make sure you have enough free disk space before downloading.
+*   **Network Connection:**  A stable and fast internet connection is essential for a smooth download.
+*   **Caching:** The Hugging Face Hub and `transformers` library use caching to avoid re-downloading models unnecessarily.  The default cache directory is usually `~/.cache/huggingface/transformers`.
+*   **File Formats:**  Models are often stored in `safetensors` or `.bin` formats.  `safetensors` is generally preferred for security and performance.
+*   **Model Card:**  Always read the model card on the Hugging Face Hub ([https://huggingface.co/microsoft/bitnet-b1.58-2B-4T](https://huggingface.co/microsoft/bitnet-b1.58-2B-4T)) for important information about the model, its intended use, limitations, and potential biases.
+*   **Gated Models:** Some models require you to accept terms of use before you can download them.  The `huggingface-cli login` command will guide you through this process if necessary.
+
+**Example Workflow (Recommended):**
+
+1.  `huggingface-cli login` (if not already logged in)
+2.  Use the `transformers` library in a Python script to download and load the model (as shown in the example above).  This is the most convenient and reliable method for most use cases.
+
+Let me know if you have any other questions or if you'd like help with a specific task related to this model!
+
+> 

{streamdown-0.17.0.dist-info → streamdown-0.19.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: streamdown
-Version: 0.17.0
+Version: 0.19.0
 Summary: A streaming markdown renderer for modern terminals with syntax highlighting
 Project-URL: Homepage, https://github.com/kristopolous/Streamdown
 Project-URL: Bug Tracker, https://github.com/kristopolous/Streamdown/issues
@@ -31,23 +31,25 @@ Description-Content-Type: text/markdown
 <img src=https://github.com/user-attachments/assets/0468eac0-2a00-4e98-82ca-09e6ac679357/>
 <br/>
 <a href=https://pypi.org/project/streamdown><img src=https://badge.fury.io/py/streamdown.svg/></a>
+<br/><strong>Terminal streaming markdown that rocks</strong>
+
 </p>
 
-**The streaming markdown renderer for the terminal that rocks**
 
-Streamdown works with [simonw's llm](https://github.com/simonw/llm) along with any other streaming markdown. You even get full readline and keyboard navigation support.
+Streamdown works with [simonw's llm](https://github.com/simonw/llm) along with any other streaming markdown, even something basic like curl. 
+It supports standard piping like any normal pager and a clean `execvp` option for robustly wrapping around interactive programs with readline or their own ANSI stuff to manage.
 ```bash
 $ pip install streamdown
 ```
 ![Streamdown is Amazing](https://github.com/user-attachments/assets/268cb340-78cc-4df0-a773-c5ac95eceeeb)
 
 ### Provides clean copyable code for long code lines
-Some *inferior* renderers inject line breaks when copying code that wraps around. We're better and now you are too!
+Other renderers inject line breaks when copying code that wraps around. We're better and now you are too!
 ![Handle That Mandle](https://github.com/user-attachments/assets/a27aa70c-f691-4796-84f0-c2eb18c7de23)
 **Tip**: You can make things prettier if you don't mind if this guarantee is broken. See the `PrettyBroken` flag below!
 
 ### Supports images
-Here's kitty and alacritty. Try to do that in glow...
+Here's kitty and alacritty. 
 ![doggie](https://github.com/user-attachments/assets/81c43983-68cd-40c1-b1d5-aa3a52004504)
 
 ### Supports hyperlinks (OSC 8) and clipboard (OSC 52)
@@ -56,9 +58,13 @@ Here's kitty and alacritty. Try to do that in glow...
 ### Supports tables
 ![table](https://github.com/user-attachments/assets/dbe3d13e-6bac-4f45-bf30-f1857ed98898)
 
-As well as everything else...
+#### As well as everything else...
 ![dunder](https://github.com/user-attachments/assets/d41d7fec-6dec-4387-b53d-f2098f269a5e)
 
+#### ...even CJK 
+Compare how streamdown wraps around and spaces this tabular Chinese description of programming languages to the same file using glow.
+![cjk](https://github.com/user-attachments/assets/b831a5f3-7ef0-48b8-8d17-bb7b605df16a)
+
 ### Colors are highly (and quickly) configurable for people who care a lot, or just a little.
 ![configurable](https://github.com/user-attachments/assets/19ca2ec9-8ea1-4a79-87ca-8352789269fe)
 

streamdown-0.19.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+streamdown/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+streamdown/sd.py,sha256=kXN5K_0pBKkDJfIthqUf8WVL1J0fqAamfl3g_mRp_VM,39330
+streamdown/ss,sha256=sel_phpaecrw6WGIHRLROsD7BFShf0rSDHheflwdUn8,277
+streamdown/ss1,sha256=CUVf86_2zeAle2oQCeTfWYqtHBrAFR_UgvptuYMQzFU,3151
+streamdown/plugins/README.md,sha256=KWqYELs9WkKJmuDzYv3cvPlZMkArsNCBUe4XDoTLjLA,1143
+streamdown/plugins/latex.py,sha256=xZMGMdx_Sw4X1piZejXFHfEG9qazU4fGeceiMI0h13Y,648
+streamdown-0.19.0.dist-info/METADATA,sha256=murXBmNpC449_wu6JFtQ5STo4br8eqGVVFD5ljCGxy0,8145
+streamdown-0.19.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+streamdown-0.19.0.dist-info/entry_points.txt,sha256=HroKFsFMGf_h9PRTE96NjvjJQWupMW5TGP5RGUr1O_Q,74
+streamdown-0.19.0.dist-info/licenses/LICENSE.MIT,sha256=SnY46EPirUsF20dZDR8HpyVgS2_4Tjxuc6f-4OdqO7U,1070
+streamdown-0.19.0.dist-info/RECORD,,

streamdown-0.17.0.dist-info/RECORD

@@ -1,10 +0,0 @@
-streamdown/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-streamdown/sd.py,sha256=jNsF77GdzZNAkTUGgpW2eq0MEXCo29BGu0DdqjqxGos,37875
-streamdown/ss,sha256=a-qosJtvfHt6cMKgib3bfGJcNkMsdWL_kWTDKjxg3po,1616
-streamdown/plugins/README.md,sha256=KWqYELs9WkKJmuDzYv3cvPlZMkArsNCBUe4XDoTLjLA,1143
-streamdown/plugins/latex.py,sha256=xZMGMdx_Sw4X1piZejXFHfEG9qazU4fGeceiMI0h13Y,648
-streamdown-0.17.0.dist-info/METADATA,sha256=glUUUfj5fsNNXARjV7zMNcP5xklvnXTxBfZ9aGRL2hA,7786
-streamdown-0.17.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-streamdown-0.17.0.dist-info/entry_points.txt,sha256=HroKFsFMGf_h9PRTE96NjvjJQWupMW5TGP5RGUr1O_Q,74
-streamdown-0.17.0.dist-info/licenses/LICENSE.MIT,sha256=SnY46EPirUsF20dZDR8HpyVgS2_4Tjxuc6f-4OdqO7U,1070
-streamdown-0.17.0.dist-info/RECORD,,