kash-shell 0.3.11__py3-none-any.whl → 0.3.13__py3-none-any.whl

kash/utils/file_utils/filename_parsing.py

@@ -1,39 +1,46 @@
 import os
+import re
 from pathlib import Path
 
 from kash.config.logger import get_logger
 from kash.utils.common.url import Url, check_if_url
-from kash.utils.errors import InvalidFilename
 from kash.utils.file_utils.file_ext import FileExt, canonicalize_file_ext
 
 log = get_logger(__name__)
 
+_valid_ext_re = re.compile(r"^[a-z0-9]*[a-z][a-z0-9]*$", re.IGNORECASE)
 
-def split_filename(path: str | Path, require_type_ext: bool = False) -> tuple[str, str, str, str]:
+
+def split_filename(path: str | Path) -> tuple[str, str, str, str]:
     """
-    Parse a filename into its path, name, (optional) type, and extension parts:
+    Parse a filename into its path, name, (optional) type, and extension parts.
+    Type and extension are optional but must be only letters/numbers and not
+    all numbers.
 
     folder/file.name.type.ext -> ("folder", "file.name", "type", "ext")
     filename.doc.txt -> ("", "filename", "note", "txt")
     filename.txt -> ("", "filename", "", "txt")
     filename -> ("", "filename", "", "")
+    filename.123.txt -> ("", "filename.123", "", "txt")
+    filename.123.456 -> ("", "filename.123.456", "", "")
     """
     path_str = str(path)
 
     dirname = os.path.dirname(path_str)
     parts = os.path.basename(path_str).rsplit(".", 2)
-    if len(parts) == 3:
+    if len(parts) == 3 and _valid_ext_re.match(parts[1]) and _valid_ext_re.match(parts[2]):
         name, item_type, ext = parts
-    elif len(parts) == 2 and not require_type_ext:
+    elif len(parts) == 3 and _valid_ext_re.match(parts[2]):
+        name = f"{parts[0]}.{parts[1]}"
+        item_type = ""
+        ext = parts[2]
+    elif len(parts) == 2 and _valid_ext_re.match(parts[1]):
         name, ext = parts
         item_type = ""
-    elif len(parts) == 1 and not require_type_ext:
-        name = parts[0]
-        item_type = ext = ""
     else:
-        raise InvalidFilename(
-            f"Filename does not match file store convention (name.type.ext): {path_str}"
-        )
+        name = os.path.basename(path_str)
+        item_type = ext = ""
+
     return dirname, name, item_type, ext
 
 
@@ -67,8 +74,6 @@ def parse_file_ext(url_or_path: str | Url | Path) -> FileExt | None:
 
 
 def test_parse_filename():
-    import pytest
-
     filename = "foo/bar/test_file.1.type.ext"
     dirname, name, item_type, ext = split_filename(filename)
     assert dirname == "foo/bar"
@@ -90,9 +95,29 @@ def test_parse_filename():
     assert item_type == ""
     assert ext == ""
 
-    filename = "missing_type.ext"
-    with pytest.raises(InvalidFilename):
-        split_filename(filename, require_type_ext=True)
+    # Numeric extensions not allowed.
+    dirname, name, item_type, ext = split_filename("test.abc")
+    assert name == "test"
+    assert ext == "abc"
+
+    dirname, name, item_type, ext = split_filename("test.123")
+    assert name == "test.123"
+    assert ext == ""
+
+    dirname, name, item_type, ext = split_filename("test.type.123")
+    assert name == "test.type.123"
+    assert item_type == ""
+    assert ext == ""
+
+    dirname, name, item_type, ext = split_filename("test.valid.123")
+    assert name == "test.valid.123"
+    assert item_type == ""
+    assert ext == ""
+
+    dirname, name, item_type, ext = split_filename("test.123.txt")
+    assert name == "test.123"
+    assert item_type == ""
+    assert ext == "txt"
 
 
 def test_parse_file_ext():

kash/{text_handling → utils/text_handling}/doc_normalization.py

@@ -1,8 +1,6 @@
 from pathlib import Path
 
-from flowmark import fill_markdown, fill_text, line_wrap_by_sentence
-from flowmark.text_filling import DEFAULT_WRAP_WIDTH
-from flowmark.text_wrapping import simple_word_splitter, wrap_paragraph
+from flowmark import fill_markdown, line_wrap_by_sentence
 from frontmatter_format import fmf_read, fmf_write
 
 from kash.utils.common.format_utils import fmt_loc
@@ -11,21 +9,31 @@ from kash.utils.file_utils.file_formats_model import Format, detect_file_format
 from kash.utils.rich_custom.ansi_cell_len import ansi_cell_len
 
 
-def normalize_formatting_ansi(text: str, format: Format | None, width=DEFAULT_WRAP_WIDTH) -> str:
+def normalize_formatting(
+    text: str,
+    format: Format | None,
+    support_ansi: bool = True,
+    cleanups: bool = True,
+) -> str:
     """
-    Normalize text formatting by wrapping lines and normalizing Markdown.
+    Normalize formatting. Currently only normalizes Markdown and leaves plaintext
+    and HTML intact.
+
+    This only does "safe" normalizations that cannot break the text.
     Enables ANSI support so ANSI codes and OSC-8 links are correctly handled.
     """
-    if format == Format.plaintext:
-        return fill_text(
-            text, width=width, word_splitter=simple_word_splitter, len_fn=ansi_cell_len
-        )
-    elif format == Format.markdown or format == Format.md_html:
+    len_fn = ansi_cell_len if support_ansi else len
+    if format == Format.markdown or format == Format.md_html:
         return fill_markdown(
             text,
-            line_wrapper=line_wrap_by_sentence(len_fn=ansi_cell_len, is_markdown=True),
-            cleanups=True,  # Safe cleanups like unbolding section headers.
+            line_wrapper=line_wrap_by_sentence(len_fn=len_fn, is_markdown=True),
+            cleanups=cleanups,
         )
+    elif format == Format.plaintext:
+        # Consider plaintext a raw format and don't normalize.
+        # We could add support for formatted plaintext as well?
+        # Then do: fill_text(text, width=width, word_splitter=simple_word_splitter, len_fn=len_fn)
+        return text
     elif format == Format.html:
         # We don't currently auto-format HTML as we sometimes use HTML with specifically chosen line breaks.
         return text
@@ -37,6 +45,7 @@ def normalize_text_file(
     path: str | Path,
     target_path: Path,
     format: Format | None = None,
+    support_ansi: bool = True,
 ) -> None:
     """
     Normalize formatting on a text file, handling Markdown, HTML, or text, as well as
@@ -48,7 +57,7 @@ def normalize_text_file(
         raise ValueError(f"Cannot format non-text files: {fmt_loc(path)}")
 
     content, metadata = fmf_read(path)
-    norm_content = normalize_formatting_ansi(content, format=format)
+    norm_content = normalize_formatting(content, format=format, support_ansi=support_ansi)
     fmf_write(not_none(target_path), norm_content, metadata)
 
 
@@ -57,6 +66,7 @@ def normalize_text_file(
 
 def test_osc8_link():
     from clideps.terminal.osc_utils import osc8_link
+    from flowmark.text_wrapping import wrap_paragraph
 
     link = osc8_link("https://example.com/" + "x" * 50, "Example")
     assert ansi_cell_len(link) == 7

kash/utils/text_handling/escape_html_tags.py

@@ -0,0 +1,156 @@
+import re
+from collections.abc import Set
+
+HTML_IN_MD_TAGS = frozenset(["div", "span", "sup", "sub", "br", "details", "summary"])
+"""These are tags that have reasonable usage in Markdown so typically would be preserved."""
+
+ALLOWED_BARE_PROTOS = frozenset(["http://", "https://", "file://"])
+
+
+def escape_html_tags(
+    html_content: str,
+    whitelist_tags: Set[str] = HTML_IN_MD_TAGS,
+    allow_bare_md_urls: bool = False,
+) -> str:
+    """
+    Escapes HTML tags by replacing '<' with '&lt;', except for whitelisted tags and
+    markdown-style URLs like <https://example.com>. Whitelist defaults to the only a
+    few common tags. But it can also be empty to escape all tags.
+    """
+    result = []
+    last_pos = 0
+
+    # Compile patterns for matching at each '<' position
+    # Match <, optional spaces, optional /, optional spaces, whitelisted tag, then optional attributes, then optional /, optional spaces, then >
+    whitelist_pattern = re.compile(
+        r"< *(/?) *(" + "|".join(whitelist_tags) + r")(?:\s+[^>]*)? *(/?) *>",
+        re.IGNORECASE,
+    )
+
+    url_pattern = None
+    if allow_bare_md_urls:
+        url_pattern = re.compile(
+            r"<(?:" + "|".join(re.escape(proto) for proto in ALLOWED_BARE_PROTOS) + r")[^>\s]+>"
+        )
+
+    # Find all '<' characters
+    for match in re.finditer(r"<", html_content):
+        start_pos = match.start()
+
+        # Add text before this '<'
+        result.append(html_content[last_pos:start_pos])
+
+        # Try to match patterns at this position
+        substring = html_content[start_pos:]
+        whitelist_match = whitelist_pattern.match(substring)
+        url_match = url_pattern and url_pattern.match(substring)
+
+        if whitelist_match:
+            result.append(whitelist_match.group(0))
+            last_pos = start_pos + len(whitelist_match.group(0))
+        elif url_match:
+            result.append(url_match.group(0))
+            last_pos = start_pos + len(url_match.group(0))
+        else:
+            # No match, escape this '<'
+            result.append("&lt;")
+            last_pos = start_pos + 1
+
+    # Add remaining text
+    result.append(html_content[last_pos:])
+
+    return "".join(result)
+
+
+## Tests
+
+
+def test_escape_html_tags():
+    """Tests the escape_html_tags function with various cases."""
+
+    # 1. Basic Whitelist Check (Default)
+    assert escape_html_tags("<div>Test</div>") == "<div>Test</div>"
+    assert escape_html_tags("<span>Test</span>") == "<span>Test</span>"
+    assert escape_html_tags("<br>") == "<br>"
+    assert (
+        escape_html_tags("<details><summary>Sum</summary>Det</details>")
+        == "<details><summary>Sum</summary>Det</details>"
+    )
+
+    # 2. Basic Escape Check
+    assert escape_html_tags("<p>Test</p>") == "&lt;p>Test&lt;/p>"
+    assert escape_html_tags("<script>alert('x');</script>") == "&lt;script>alert('x');&lt;/script>"
+    assert escape_html_tags("<img>") == "&lt;img>"
+
+    # 3. Case Insensitivity
+    assert escape_html_tags("<DiV>Case</DiV>") == "<DiV>Case</DiV>"  # Whitelisted
+    assert escape_html_tags("<P>Test</P>") == "&lt;P>Test&lt;/P>"  # Escaped
+
+    # 4. Self-closing tags
+    assert escape_html_tags("<br/>") == "<br/>"  # Whitelisted
+    assert escape_html_tags("<br />") == "<br />"  # Whitelisted
+    assert escape_html_tags("<img/>") == "&lt;img/>"  # Escaped
+
+    # 5. Tags with Attributes
+    assert (
+        escape_html_tags('<div class="foo">Test</div>') == '<div class="foo">Test</div>'
+    )  # Whitelisted
+    assert (
+        escape_html_tags('<span id="bar" data-val="x">Test</span>')
+        == '<span id="bar" data-val="x">Test</span>'
+    )  # Whitelisted
+    assert escape_html_tags('<p class="foo">Test</p>') == '&lt;p class="foo">Test&lt;/p>'  # Escaped
+    assert escape_html_tags('<img src="a.jpg"/>') == '&lt;img src="a.jpg"/>'  # Escaped
+
+    # 6. Markdown URL Handling
+    url_md = "Check <https://example.com> and <http://test.org/path>"
+    assert escape_html_tags(url_md, allow_bare_md_urls=True) == url_md
+    assert (
+        escape_html_tags(url_md, allow_bare_md_urls=False)
+        == "Check &lt;https://example.com> and &lt;http://test.org/path>"
+    )
+
+    url_mixed = "<div>Link: <https://ok.com></div> <script>no</script>"
+    expected_mixed_urls_allowed = "<div>Link: <https://ok.com></div> &lt;script>no&lt;/script>"
+    expected_mixed_urls_disallowed = (
+        "<div>Link: &lt;https://ok.com></div> &lt;script>no&lt;/script>"
+    )
+    assert escape_html_tags(url_mixed, allow_bare_md_urls=True) == expected_mixed_urls_allowed
+    assert escape_html_tags(url_mixed, allow_bare_md_urls=False) == expected_mixed_urls_disallowed
+
+    assert (
+        escape_html_tags("<http://malformed url>", allow_bare_md_urls=True)
+        == "&lt;http://malformed url>"
+    )
+    assert (
+        escape_html_tags("</https://example.com>", allow_bare_md_urls=True)
+        == "&lt;/https://example.com>"
+    )  # Closing URL-like is escaped
+
+    # 7. Nested/Malformed '<' and Edge Cases
+    assert escape_html_tags("<<script>>") == "&lt;&lt;script>>"  # Escaped non-tag <
+    assert escape_html_tags("<div><p>nested</p></div>") == "<div>&lt;p>nested&lt;/p></div>"
+    assert escape_html_tags("<div<span") == "&lt;div&lt;span"  # Incomplete tags are escaped
+    assert (
+        escape_html_tags("Text < with > inside") == "Text &lt; with > inside"
+    )  # Escape < even if > exists later
+    assert escape_html_tags("<") == "&lt;"
+    assert escape_html_tags(">") == ">"
+    assert escape_html_tags("<>") == "&lt;>"
+    assert escape_html_tags("< >") == "&lt; >"
+    assert escape_html_tags("< / div >") == "< / div >"  # Whitelisted closing tag with spaces
+
+    # 8. Mixed Content Combination
+    complex_html = "<DiV class='A'>Hello <Br/> <p>World</p> <https://link.com> </DiV>"
+    expected_complex_allowed = (
+        "<DiV class='A'>Hello <Br/> &lt;p>World&lt;/p> <https://link.com> </DiV>"
+    )
+    expected_complex_disallowed = (
+        "<DiV class='A'>Hello <Br/> &lt;p>World&lt;/p> &lt;https://link.com> </DiV>"
+    )
+    assert escape_html_tags(complex_html, allow_bare_md_urls=True) == expected_complex_allowed
+    assert escape_html_tags(complex_html, allow_bare_md_urls=False) == expected_complex_disallowed
+
+    # 9. Empty/No Tags
+    assert escape_html_tags("") == ""
+    assert escape_html_tags("Just plain text, no tags.") == "Just plain text, no tags."

kash/{text_handling → utils/text_handling}/markdown_utils.py

@@ -1,15 +1,15 @@
 import re
-from typing import Any
+from textwrap import dedent
+from typing import Any, TypeAlias
 
 import marko
 import regex
 from marko.block import Heading, ListItem
 from marko.inline import Link
 
-from kash.config.logger import get_logger
 from kash.utils.common.url import Url
 
-log = get_logger(__name__)
+HTag: TypeAlias = str
 
 # Characters that commonly need escaping in Markdown inline text.
 MARKDOWN_ESCAPE_CHARS = r"([\\`*_{}\[\]()#+.!-])"
@@ -128,7 +128,7 @@ def extract_bullet_points(content: str) -> list[str]:
     return _tree_bullet_points(document)
 
 
-def _type_from_heading(heading: Heading) -> str:
+def _type_from_heading(heading: Heading) -> HTag:
     if heading.level in [1, 2, 3, 4, 5, 6]:
         return f"h{heading.level}"
     else:
@@ -174,6 +174,43 @@ def find_markdown_text(
         pos = match.end()
 
 
+def extract_headings(text: str) -> list[tuple[HTag, str]]:
+    """
+    Extract all Markdown headings from the given content.
+    Returns a list of (tag, text) tuples:
+    [("h1", "Main Title"), ("h2", "Subtitle")]
+    where `#` corresponds to `h1`, `##` to `h2`, etc.
+    """
+    document = marko.parse(text)
+    headings_list: list[tuple[HTag, str]] = []
+
+    def _collect_headings_recursive(element: Any) -> None:
+        if isinstance(element, Heading):
+            tag = _type_from_heading(element)
+            content = _extract_text(element).strip()
+            headings_list.append((tag, content))
+
+        if hasattr(element, "children"):
+            for child in element.children:
+                _collect_headings_recursive(child)
+
+    _collect_headings_recursive(document)
+
+    return headings_list
+
+
+def first_heading(text: str, *, allowed_tags: tuple[HTag, ...] = ("h1", "h2")) -> str | None:
+    """
+    Find the text of the first heading. Returns first h1 if present, otherwise first h2, etc.
+    """
+    headings = extract_headings(text)
+    for goal_tag in allowed_tags:
+        for h_tag, h_text in headings:
+            if h_tag == goal_tag:
+                return h_text
+    return None
+
+
 ## Tests
 
 
@@ -224,3 +261,44 @@ def test_find_markdown_text() -> None:  # pragma: no cover
     pattern = re.compile("bar", re.IGNORECASE)
     match = find_markdown_text(pattern, text)
     assert match is None
+
+
+def test_extract_headings_and_first_header() -> None:
+    markdown_content = dedent("""
+        # Title 1
+        Some text.
+        ## Subtitle 1.1
+        More text.
+        ### Sub-subtitle 1.1.1
+        Even more text.
+        # Title 2 *with formatting*
+        And final text.
+        ## Subtitle 2.1
+        """)
+    expected_headings = [
+        ("h1", "Title 1"),
+        ("h2", "Subtitle 1.1"),
+        ("h3", "Sub-subtitle 1.1.1"),
+        ("h1", "Title 2 with formatting"),
+        ("h2", "Subtitle 2.1"),
+    ]
+    assert extract_headings(markdown_content) == expected_headings
+
+    assert first_heading(markdown_content) == "Title 1"
+    assert first_heading(markdown_content) == "Title 1"
+    assert first_heading(markdown_content, allowed_tags=("h2",)) == "Subtitle 1.1"
+    assert first_heading(markdown_content, allowed_tags=("h3",)) == "Sub-subtitle 1.1.1"
+    assert first_heading(markdown_content, allowed_tags=("h4",)) is None
+
+    assert extract_headings("") == []
+    assert first_heading("") is None
+    assert first_heading("Just text, no headers.") is None
+
+    markdown_h2_only = "## Only H2 Here"
+    assert extract_headings(markdown_h2_only) == [("h2", "Only H2 Here")]
+    assert first_heading(markdown_h2_only) == "Only H2 Here"
+    assert first_heading(markdown_h2_only, allowed_tags=("h2",)) == "Only H2 Here"
+
+    formatted_header_md = "## *Formatted* _Header_ [link](#anchor)"
+    assert extract_headings(formatted_header_md) == [("h2", "Formatted Header link")]
+    assert first_heading(formatted_header_md, allowed_tags=("h2",)) == "Formatted Header link"

kash/utils/text_handling/markdownify_utils.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import re
+
+from kash.utils.text_handling.escape_html_tags import escape_html_tags
+
+_single_tilde_pat = re.compile(r"(?<!~)~(?!~)")
+_alt_tilde = "～"
+
+
+def _fix_single_tilde(html: str) -> str:
+    """
+    Escape standalone ~ characters with spaces before/after to avoid
+    misinterpretation by markdownify as strikethrough. Using ～ because it's
+    hard to properly escape ~ in a way that markdownify will respect.
+    """
+
+    def replace_tilde(match: re.Match[str]) -> str:
+        start = match.start()
+        end = match.end()
+        # Check for space before or after
+        has_space_before = start > 0 and html[start - 1].isspace()
+        has_space_after = end < len(html) and html[end].isspace()
+        return _alt_tilde if has_space_before or has_space_after else "~"
+
+    return _single_tilde_pat.sub(replace_tilde, html)
+
+
+def markdownify_preprocess(html: str) -> str:
+    """
+    Preprocess HTML before passing it to markdownify.
+    """
+    return _fix_single_tilde(html)
+
+
+# Good options for markdownify. Without setting sup_symbol and sub_symbol, that
+# info is typically lost.
+MARKDOWNIFY_OPTIONS = {
+    "sup_symbol": "<__sup>",
+    "sub_symbol": "<__sub>",
+    "escape_underscores": True,
+    "escape_asterisks": True,
+    "escape_misc": False,  # This suppresses gratuitous escaping of -, ., etc.
+    "newline_style": "BACKSLASH",
+}
+
+
+def _escape_html_in_md(md_text: str, whitelist_tags: set[str] | None = None) -> str:
+    """
+    HTML tags originally escaped with entities can get parsed and appear unescaped
+    in the Markdown so it usually makes sense to do a full escaping (except for our
+    custom sup/sub tags).
+    """
+    # Output from markdownify (especially from docx or other conversions) should
+    # not have any HTML tags except for the custom sup/sub tags we've added.
+    return escape_html_tags(
+        md_text,
+        allow_bare_md_urls=True,
+        whitelist_tags={"__sup", "__sub"} | (whitelist_tags or set()),
+    )
+
+
+def markdownify_postprocess(md_text: str) -> str:
+    """
+    Postprocess Markdown after markdownify has converted HTML to Markdown.
+    """
+    md_text = _escape_html_in_md(md_text)
+    # We use our own custom tags for sup/sub to avoid possible conflicts with other
+    # tags in a doc. But when done we should replace them with the standard ones.
+    return (
+        md_text.replace("<__sup>", "<sup>")
+        .replace("</__sup>", "</sup>")
+        .replace("<__sub>", "<sub>")
+        .replace("</__sub>", "</sub>")
+    )
+
+
+def markdownify_custom(html: str) -> str:
+    """
+    Customized version of `markdownify_convert to be more robust than with default settings.
+    """
+
+    from markdownify import markdownify as markdownify_convert
+
+    preprocessed_html = markdownify_preprocess(html)
+    md_text = markdownify_convert(preprocessed_html, **MARKDOWNIFY_OPTIONS)
+    return markdownify_postprocess(md_text)

kash/{text_handling → utils/text_handling}/unified_diffs.py

@@ -6,15 +6,6 @@ from funlog import abbreviate_arg
 from patch_ng import PatchSet
 from pydantic.dataclasses import dataclass
 
-from kash.config.logger import get_logger
-from kash.model.items_model import Item, ItemRelations, ItemType
-from kash.model.paths_model import StorePath
-from kash.utils.errors import ContentError
-from kash.utils.file_utils.file_formats_model import Format
-
-log = get_logger(__name__)
-
-
 # TODO: Support diffs of path lists as well, including renames and moves.
 
 
@@ -77,7 +68,7 @@ def unified_diff(
 
     patch_set = PatchSet(BytesIO(diff_text.encode("utf-8")))
     if patch_set.errors > 0:
-        raise ContentError(
+        raise ValueError(
             f"Had {patch_set.errors} errors parsing diff of `{from_name}` and `{to_name}`: {abbreviate_arg(diff_text)}"
         )
 
@@ -102,37 +93,3 @@ def unified_diff_files(from_file: str | Path, to_file: str | Path) -> UnifiedDif
         content2 = f2.read()
 
     return unified_diff(content1, content2, from_name, to_name)
-
-
-def unified_diff_items(from_item: Item, to_item: Item, strict: bool = True) -> Item:
-    """
-    Generate a unified diff between two items. If `strict` is true, will raise
-    an error if the items are of different formats.
-    """
-    if not from_item.body and not to_item.body:
-        raise ContentError(f"No body to diff for {from_item} and {to_item}")
-    if not from_item.store_path or not to_item.store_path:
-        raise ContentError("No store path on items; save before diffing")
-    diff_items = [item for item in [from_item, to_item] if item.format == Format.diff]
-    if len(diff_items) == 1:
-        raise ContentError(
-            f"Cannot compare diffs to non-diffs: {from_item.format}, {to_item.format}"
-        )
-    if len(diff_items) > 0 or from_item.format != to_item.format:
-        msg = f"Diffing items of incompatible format: {from_item.format}, {to_item.format}"
-        if strict:
-            raise ContentError(msg)
-        else:
-            log.warning("%s", msg)
-
-    from_path, to_path = StorePath(from_item.store_path), StorePath(to_item.store_path)
-
-    diff = unified_diff(from_item.body, to_item.body, str(from_path), str(to_path))
-
-    return Item(
-        type=ItemType.doc,
-        title=f"Diff of {from_path} and {to_path}",
-        format=Format.diff,
-        relations=ItemRelations(diff_of=[from_path, to_path]),
-        body=diff.patch_text,
-    )