kash-shell 0.3.12__py3-none-any.whl → 0.3.14__py3-none-any.whl

kash/utils/file_utils/file_formats_model.py

@@ -4,7 +4,7 @@ from dataclasses import dataclass
 from enum import Enum
 from pathlib import Path
 
-from kash.utils.common.url import Url, is_file_url, parse_file_url
+from kash.utils.common.url import is_valid_path
 from kash.utils.file_utils.file_ext import FileExt
 from kash.utils.file_utils.file_formats import (
     MIME_EMPTY,
@@ -112,6 +112,9 @@ class Format(Enum):
 
     @property
     def is_doc(self) -> bool:
+        """
+        Is this a textual document of some kind?
+        """
         return self in [
             self.markdown,
             self.md_html,
@@ -119,6 +122,7 @@ class Format(Enum):
             self.pdf,
             self.docx,
             self.pptx,
+            self.epub,
         ]
 
     @property
@@ -139,7 +143,13 @@ class Format(Enum):
 
     @property
     def is_markdown(self) -> bool:
-        return self in [self.markdown, self.md_html]
+        """Is in pure Markdown (no HTML)."""
+        return self in [self.markdown]
+
+    @property
+    def is_markdown_with_html(self) -> bool:
+        """Is in Markdown with HTML."""
+        return self in [self.md_html]
 
     @property
     def is_html(self) -> bool:
@@ -340,8 +350,8 @@ Format._init_mime_type_map()
 
 @dataclass(frozen=True)
 class FileFormatInfo:
-    file_ext: FileExt | None
-    """File extension, if recognized."""
+    current_file_ext: FileExt | None
+    """File extension, if recognized and in the current filename."""
 
     format: Format | None
     """Format, if recognized."""
@@ -349,11 +359,18 @@ class FileFormatInfo:
     mime_type: MimeType | None
     """Raw mime type, which may include more formats than the ones above."""
 
+    @property
+    def suggested_file_ext(self) -> FileExt | None:
+        """
+        Suggested file extension based on detected format.
+        """
+        return self.format.file_ext if self.format else self.current_file_ext
+
     @property
     def is_text(self) -> bool:
         return bool(
-            self.file_ext
-            and self.file_ext.is_text
+            self.current_file_ext
+            and self.current_file_ext.is_text
             or self.format
             and self.format.is_text
             or self.mime_type
@@ -373,8 +390,8 @@ class FileFormatInfo:
     @property
     def is_image(self) -> bool:
         return bool(
-            self.file_ext
-            and self.file_ext.is_image
+            self.current_file_ext
+            and self.current_file_ext.is_image
             or self.format
             and self.format.is_image
             or self.mime_type
@@ -395,15 +412,6 @@ class FileFormatInfo:
         return self.as_str()
 
 
-def _guess_format(file_ext: FileExt | None, mime_type: MimeType | None) -> Format | None:
-    format = None
-    if file_ext:
-        format = Format.guess_by_file_ext(file_ext)
-    if not format and mime_type:
-        format = Format.from_mime_type(mime_type)
-    return format
-
-
 def guess_format_by_name(path: str | Path) -> Format | None:
     """
     Fast guess of file format by the file name only.
@@ -412,22 +420,39 @@ def guess_format_by_name(path: str | Path) -> Format | None:
     return Format.guess_by_file_ext(file_ext) if file_ext else None
 
 
-def file_format_info(path: str | Path, always_check_content: bool = False) -> FileFormatInfo:
+def file_format_info(
+    path: str | Path,
+    suggested_mime_type: MimeType | None = None,
+) -> FileFormatInfo:
     """
     Get info on the file format path and content (file extension and file content).
     Looks at the file extension first and then the file content if needed.
-    If `always_check_content` is True, look at the file content even if we
-    recognize the file extension.
+    If `suggested_mime_type` is provided, it will be used as the detected mime type
+    instead of detecting it from the file content.
     """
+    if not is_valid_path(path):
+        raise ValueError(f"Expected a file path but got: {path!r}")
+
     path = Path(path)
     file_ext = parse_file_ext(path)
-    if always_check_content or not file_ext:
+    if not suggested_mime_type and not file_ext:
         # Look at the file content.
         detected_mime_type = detect_mime_type(path)
+    elif suggested_mime_type:
+        detected_mime_type = suggested_mime_type
     else:
         detected_mime_type = None
-    format = _guess_format(file_ext, detected_mime_type)
+
+    # Pick format first by file extension, then by detected mime type.
+    format = None
+    if file_ext:
+        format = Format.guess_by_file_ext(file_ext)
+    if not format and detected_mime_type:
+        format = Format.from_mime_type(detected_mime_type)
+
+    # Attempt to canonicalize the mime type to match the format.
     final_mime_type = format.mime_type if format else detected_mime_type
+
     return FileFormatInfo(file_ext, format, final_mime_type)
 
 
@@ -445,26 +470,3 @@ def detect_media_type(filename: str | Path) -> MediaType:
     fmt = detect_file_format(filename)
     media_type = fmt.media_type if fmt else MediaType.binary
     return media_type
-
-
-def choose_file_ext(url_or_path: Url | Path | str) -> FileExt | None:
-    """
-    Pick a suffix to reflect the type of the content. Recognizes known file
-    extensions, then tries libmagic, then gives up.
-    """
-
-    def file_ext_for(path: Path) -> FileExt | None:
-        fmt = detect_file_format(path)
-        return fmt.file_ext if fmt else None
-
-    ext = None
-    if isinstance(url_or_path, Path):
-        ext = parse_file_ext(url_or_path) or file_ext_for(url_or_path)
-    elif is_file_url(url_or_path):
-        path = parse_file_url(url_or_path)
-        if path:
-            ext = parse_file_ext(path) or file_ext_for(path)
-    else:
-        ext = parse_file_ext(url_or_path)
-
-    return ext

kash/utils/file_utils/filename_parsing.py

@@ -1,39 +1,46 @@
 import os
+import re
 from pathlib import Path
 
 from kash.config.logger import get_logger
 from kash.utils.common.url import Url, check_if_url
-from kash.utils.errors import InvalidFilename
 from kash.utils.file_utils.file_ext import FileExt, canonicalize_file_ext
 
 log = get_logger(__name__)
 
+_valid_ext_re = re.compile(r"^[a-z0-9]*[a-z][a-z0-9]*$", re.IGNORECASE)
 
-def split_filename(path: str | Path, require_type_ext: bool = False) -> tuple[str, str, str, str]:
+
+def split_filename(path: str | Path) -> tuple[str, str, str, str]:
     """
-    Parse a filename into its path, name, (optional) type, and extension parts:
+    Parse a filename into its path, name, (optional) type, and extension parts.
+    Type and extension are optional but must be only letters/numbers and not
+    all numbers.
 
     folder/file.name.type.ext -> ("folder", "file.name", "type", "ext")
     filename.doc.txt -> ("", "filename", "note", "txt")
     filename.txt -> ("", "filename", "", "txt")
     filename -> ("", "filename", "", "")
+    filename.123.txt -> ("", "filename.123", "", "txt")
+    filename.123.456 -> ("", "filename.123.456", "", "")
     """
     path_str = str(path)
 
     dirname = os.path.dirname(path_str)
     parts = os.path.basename(path_str).rsplit(".", 2)
-    if len(parts) == 3:
+    if len(parts) == 3 and _valid_ext_re.match(parts[1]) and _valid_ext_re.match(parts[2]):
         name, item_type, ext = parts
-    elif len(parts) == 2 and not require_type_ext:
+    elif len(parts) == 3 and _valid_ext_re.match(parts[2]):
+        name = f"{parts[0]}.{parts[1]}"
+        item_type = ""
+        ext = parts[2]
+    elif len(parts) == 2 and _valid_ext_re.match(parts[1]):
         name, ext = parts
         item_type = ""
-    elif len(parts) == 1 and not require_type_ext:
-        name = parts[0]
-        item_type = ext = ""
     else:
-        raise InvalidFilename(
-            f"Filename does not match file store convention (name.type.ext): {path_str}"
-        )
+        name = os.path.basename(path_str)
+        item_type = ext = ""
+
     return dirname, name, item_type, ext
 
 
@@ -67,8 +74,6 @@ def parse_file_ext(url_or_path: str | Url | Path) -> FileExt | None:
 
 
 def test_parse_filename():
-    import pytest
-
     filename = "foo/bar/test_file.1.type.ext"
     dirname, name, item_type, ext = split_filename(filename)
     assert dirname == "foo/bar"
@@ -90,9 +95,29 @@ def test_parse_filename():
     assert item_type == ""
     assert ext == ""
 
-    filename = "missing_type.ext"
-    with pytest.raises(InvalidFilename):
-        split_filename(filename, require_type_ext=True)
+    # Numeric extensions not allowed.
+    dirname, name, item_type, ext = split_filename("test.abc")
+    assert name == "test"
+    assert ext == "abc"
+
+    dirname, name, item_type, ext = split_filename("test.123")
+    assert name == "test.123"
+    assert ext == ""
+
+    dirname, name, item_type, ext = split_filename("test.type.123")
+    assert name == "test.type.123"
+    assert item_type == ""
+    assert ext == ""
+
+    dirname, name, item_type, ext = split_filename("test.valid.123")
+    assert name == "test.valid.123"
+    assert item_type == ""
+    assert ext == ""
+
+    dirname, name, item_type, ext = split_filename("test.123.txt")
+    assert name == "test.123"
+    assert item_type == ""
+    assert ext == "txt"
 
 
 def test_parse_file_ext():

kash/{text_handling → utils/text_handling}/doc_normalization.py

@@ -1,8 +1,6 @@
 from pathlib import Path
 
-from flowmark import fill_markdown, fill_text, line_wrap_by_sentence
-from flowmark.text_filling import DEFAULT_WRAP_WIDTH
-from flowmark.text_wrapping import simple_word_splitter
+from flowmark import fill_markdown, line_wrap_by_sentence
 from frontmatter_format import fmf_read, fmf_write
 
 from kash.utils.common.format_utils import fmt_loc
@@ -14,24 +12,28 @@ from kash.utils.rich_custom.ansi_cell_len import ansi_cell_len
 def normalize_formatting(
     text: str,
     format: Format | None,
-    width=DEFAULT_WRAP_WIDTH,
     support_ansi: bool = True,
     cleanups: bool = True,
 ) -> str:
     """
-    Normalize text formatting by wrapping lines and normalizing Markdown.
+    Normalize formatting. Currently only normalizes Markdown and leaves plaintext
+    and HTML intact.
+
     This only does "safe" normalizations that cannot break the text.
     Enables ANSI support so ANSI codes and OSC-8 links are correctly handled.
     """
     len_fn = ansi_cell_len if support_ansi else len
-    if format == Format.plaintext:
-        return fill_text(text, width=width, word_splitter=simple_word_splitter, len_fn=len_fn)
-    elif format == Format.markdown or format == Format.md_html:
+    if format == Format.markdown or format == Format.md_html:
         return fill_markdown(
             text,
             line_wrapper=line_wrap_by_sentence(len_fn=len_fn, is_markdown=True),
             cleanups=cleanups,
         )
+    elif format == Format.plaintext:
+        # Consider plaintext a raw format and don't normalize.
+        # We could add support for formatted plaintext as well?
+        # Then do: fill_text(text, width=width, word_splitter=simple_word_splitter, len_fn=len_fn)
+        return text
     elif format == Format.html:
         # We don't currently auto-format HTML as we sometimes use HTML with specifically chosen line breaks.
         return text

kash/utils/text_handling/escape_html_tags.py

@@ -0,0 +1,156 @@
+import re
+from collections.abc import Set
+
+HTML_IN_MD_TAGS = frozenset(["div", "span", "sup", "sub", "br", "details", "summary"])
+"""These are tags that have reasonable usage in Markdown so typically would be preserved."""
+
+ALLOWED_BARE_PROTOS = frozenset(["http://", "https://", "file://"])
+
+
+def escape_html_tags(
+    html_content: str,
+    whitelist_tags: Set[str] = HTML_IN_MD_TAGS,
+    allow_bare_md_urls: bool = False,
+) -> str:
+    """
+    Escapes HTML tags by replacing '<' with '&lt;', except for whitelisted tags and
+    markdown-style URLs like <https://example.com>. Whitelist defaults to the only a
+    few common tags. But it can also be empty to escape all tags.
+    """
+    result = []
+    last_pos = 0
+
+    # Compile patterns for matching at each '<' position
+    # Match <, optional spaces, optional /, optional spaces, whitelisted tag, then optional attributes, then optional /, optional spaces, then >
+    whitelist_pattern = re.compile(
+        r"< *(/?) *(" + "|".join(whitelist_tags) + r")(?:\s+[^>]*)? *(/?) *>",
+        re.IGNORECASE,
+    )
+
+    url_pattern = None
+    if allow_bare_md_urls:
+        url_pattern = re.compile(
+            r"<(?:" + "|".join(re.escape(proto) for proto in ALLOWED_BARE_PROTOS) + r")[^>\s]+>"
+        )
+
+    # Find all '<' characters
+    for match in re.finditer(r"<", html_content):
+        start_pos = match.start()
+
+        # Add text before this '<'
+        result.append(html_content[last_pos:start_pos])
+
+        # Try to match patterns at this position
+        substring = html_content[start_pos:]
+        whitelist_match = whitelist_pattern.match(substring)
+        url_match = url_pattern and url_pattern.match(substring)
+
+        if whitelist_match:
+            result.append(whitelist_match.group(0))
+            last_pos = start_pos + len(whitelist_match.group(0))
+        elif url_match:
+            result.append(url_match.group(0))
+            last_pos = start_pos + len(url_match.group(0))
+        else:
+            # No match, escape this '<'
+            result.append("&lt;")
+            last_pos = start_pos + 1
+
+    # Add remaining text
+    result.append(html_content[last_pos:])
+
+    return "".join(result)
+
+
+## Tests
+
+
+def test_escape_html_tags():
+    """Tests the escape_html_tags function with various cases."""
+
+    # 1. Basic Whitelist Check (Default)
+    assert escape_html_tags("<div>Test</div>") == "<div>Test</div>"
+    assert escape_html_tags("<span>Test</span>") == "<span>Test</span>"
+    assert escape_html_tags("<br>") == "<br>"
+    assert (
+        escape_html_tags("<details><summary>Sum</summary>Det</details>")
+        == "<details><summary>Sum</summary>Det</details>"
+    )
+
+    # 2. Basic Escape Check
+    assert escape_html_tags("<p>Test</p>") == "&lt;p>Test&lt;/p>"
+    assert escape_html_tags("<script>alert('x');</script>") == "&lt;script>alert('x');&lt;/script>"
+    assert escape_html_tags("<img>") == "&lt;img>"
+
+    # 3. Case Insensitivity
+    assert escape_html_tags("<DiV>Case</DiV>") == "<DiV>Case</DiV>"  # Whitelisted
+    assert escape_html_tags("<P>Test</P>") == "&lt;P>Test&lt;/P>"  # Escaped
+
+    # 4. Self-closing tags
+    assert escape_html_tags("<br/>") == "<br/>"  # Whitelisted
+    assert escape_html_tags("<br />") == "<br />"  # Whitelisted
+    assert escape_html_tags("<img/>") == "&lt;img/>"  # Escaped
+
+    # 5. Tags with Attributes
+    assert (
+        escape_html_tags('<div class="foo">Test</div>') == '<div class="foo">Test</div>'
+    )  # Whitelisted
+    assert (
+        escape_html_tags('<span id="bar" data-val="x">Test</span>')
+        == '<span id="bar" data-val="x">Test</span>'
+    )  # Whitelisted
+    assert escape_html_tags('<p class="foo">Test</p>') == '&lt;p class="foo">Test&lt;/p>'  # Escaped
+    assert escape_html_tags('<img src="a.jpg"/>') == '&lt;img src="a.jpg"/>'  # Escaped
+
+    # 6. Markdown URL Handling
+    url_md = "Check <https://example.com> and <http://test.org/path>"
+    assert escape_html_tags(url_md, allow_bare_md_urls=True) == url_md
+    assert (
+        escape_html_tags(url_md, allow_bare_md_urls=False)
+        == "Check &lt;https://example.com> and &lt;http://test.org/path>"
+    )
+
+    url_mixed = "<div>Link: <https://ok.com></div> <script>no</script>"
+    expected_mixed_urls_allowed = "<div>Link: <https://ok.com></div> &lt;script>no&lt;/script>"
+    expected_mixed_urls_disallowed = (
+        "<div>Link: &lt;https://ok.com></div> &lt;script>no&lt;/script>"
+    )
+    assert escape_html_tags(url_mixed, allow_bare_md_urls=True) == expected_mixed_urls_allowed
+    assert escape_html_tags(url_mixed, allow_bare_md_urls=False) == expected_mixed_urls_disallowed
+
+    assert (
+        escape_html_tags("<http://malformed url>", allow_bare_md_urls=True)
+        == "&lt;http://malformed url>"
+    )
+    assert (
+        escape_html_tags("</https://example.com>", allow_bare_md_urls=True)
+        == "&lt;/https://example.com>"
+    )  # Closing URL-like is escaped
+
+    # 7. Nested/Malformed '<' and Edge Cases
+    assert escape_html_tags("<<script>>") == "&lt;&lt;script>>"  # Escaped non-tag <
+    assert escape_html_tags("<div><p>nested</p></div>") == "<div>&lt;p>nested&lt;/p></div>"
+    assert escape_html_tags("<div<span") == "&lt;div&lt;span"  # Incomplete tags are escaped
+    assert (
+        escape_html_tags("Text < with > inside") == "Text &lt; with > inside"
+    )  # Escape < even if > exists later
+    assert escape_html_tags("<") == "&lt;"
+    assert escape_html_tags(">") == ">"
+    assert escape_html_tags("<>") == "&lt;>"
+    assert escape_html_tags("< >") == "&lt; >"
+    assert escape_html_tags("< / div >") == "< / div >"  # Whitelisted closing tag with spaces
+
+    # 8. Mixed Content Combination
+    complex_html = "<DiV class='A'>Hello <Br/> <p>World</p> <https://link.com> </DiV>"
+    expected_complex_allowed = (
+        "<DiV class='A'>Hello <Br/> &lt;p>World&lt;/p> <https://link.com> </DiV>"
+    )
+    expected_complex_disallowed = (
+        "<DiV class='A'>Hello <Br/> &lt;p>World&lt;/p> &lt;https://link.com> </DiV>"
+    )
+    assert escape_html_tags(complex_html, allow_bare_md_urls=True) == expected_complex_allowed
+    assert escape_html_tags(complex_html, allow_bare_md_urls=False) == expected_complex_disallowed
+
+    # 9. Empty/No Tags
+    assert escape_html_tags("") == ""
+    assert escape_html_tags("Just plain text, no tags.") == "Just plain text, no tags."

kash/{text_handling → utils/text_handling}/markdown_utils.py

@@ -7,11 +7,8 @@ import regex
 from marko.block import Heading, ListItem
 from marko.inline import Link
 
-from kash.config.logger import get_logger
 from kash.utils.common.url import Url
 
-log = get_logger(__name__)
-
 HTag: TypeAlias = str
 
 # Characters that commonly need escaping in Markdown inline text.

kash/utils/text_handling/markdownify_utils.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import re
+
+from kash.utils.text_handling.escape_html_tags import escape_html_tags
+
+_single_tilde_pat = re.compile(r"(?<!~)~(?!~)")
+_alt_tilde = "～"
+
+
+def _fix_single_tilde(html: str) -> str:
+    """
+    Escape standalone ~ characters with spaces before/after to avoid
+    misinterpretation by markdownify as strikethrough. Using ～ because it's
+    hard to properly escape ~ in a way that markdownify will respect.
+    """
+
+    def replace_tilde(match: re.Match[str]) -> str:
+        start = match.start()
+        end = match.end()
+        # Check for space before or after
+        has_space_before = start > 0 and html[start - 1].isspace()
+        has_space_after = end < len(html) and html[end].isspace()
+        return _alt_tilde if has_space_before or has_space_after else "~"
+
+    return _single_tilde_pat.sub(replace_tilde, html)
+
+
+def markdownify_preprocess(html: str) -> str:
+    """
+    Preprocess HTML before passing it to markdownify.
+    """
+    return _fix_single_tilde(html)
+
+
+# Good options for markdownify. Without setting sup_symbol and sub_symbol, that
+# info is typically lost.
+MARKDOWNIFY_OPTIONS = {
+    "sup_symbol": "<__sup>",
+    "sub_symbol": "<__sub>",
+    "escape_underscores": True,
+    "escape_asterisks": True,
+    "escape_misc": False,  # This suppresses gratuitous escaping of -, ., etc.
+    "newline_style": "BACKSLASH",
+}
+
+
+def _escape_html_in_md(md_text: str, whitelist_tags: set[str] | None = None) -> str:
+    """
+    HTML tags originally escaped with entities can get parsed and appear unescaped
+    in the Markdown so it usually makes sense to do a full escaping (except for our
+    custom sup/sub tags).
+    """
+    # Output from markdownify (especially from docx or other conversions) should
+    # not have any HTML tags except for the custom sup/sub tags we've added.
+    return escape_html_tags(
+        md_text,
+        allow_bare_md_urls=True,
+        whitelist_tags={"__sup", "__sub"} | (whitelist_tags or set()),
+    )
+
+
+def markdownify_postprocess(md_text: str) -> str:
+    """
+    Postprocess Markdown after markdownify has converted HTML to Markdown.
+    """
+    md_text = _escape_html_in_md(md_text)
+    # We use our own custom tags for sup/sub to avoid possible conflicts with other
+    # tags in a doc. But when done we should replace them with the standard ones.
+    return (
+        md_text.replace("<__sup>", "<sup>")
+        .replace("</__sup>", "</sup>")
+        .replace("<__sub>", "<sub>")
+        .replace("</__sub>", "</sub>")
+    )
+
+
+def markdownify_custom(html: str) -> str:
+    """
+    Customized version of `markdownify_convert to be more robust than with default settings.
+    """
+
+    from markdownify import markdownify as markdownify_convert
+
+    preprocessed_html = markdownify_preprocess(html)
+    md_text = markdownify_convert(preprocessed_html, **MARKDOWNIFY_OPTIONS)
+    return markdownify_postprocess(md_text)

kash/{text_handling → utils/text_handling}/unified_diffs.py

@@ -6,15 +6,6 @@ from funlog import abbreviate_arg
 from patch_ng import PatchSet
 from pydantic.dataclasses import dataclass
 
-from kash.config.logger import get_logger
-from kash.model.items_model import Item, ItemRelations, ItemType
-from kash.model.paths_model import StorePath
-from kash.utils.errors import ContentError
-from kash.utils.file_utils.file_formats_model import Format
-
-log = get_logger(__name__)
-
-
 # TODO: Support diffs of path lists as well, including renames and moves.
 
 
@@ -77,7 +68,7 @@ def unified_diff(
 
     patch_set = PatchSet(BytesIO(diff_text.encode("utf-8")))
     if patch_set.errors > 0:
-        raise ContentError(
+        raise ValueError(
             f"Had {patch_set.errors} errors parsing diff of `{from_name}` and `{to_name}`: {abbreviate_arg(diff_text)}"
         )
 
@@ -102,37 +93,3 @@ def unified_diff_files(from_file: str | Path, to_file: str | Path) -> UnifiedDif
         content2 = f2.read()
 
     return unified_diff(content1, content2, from_name, to_name)
-
-
-def unified_diff_items(from_item: Item, to_item: Item, strict: bool = True) -> Item:
-    """
-    Generate a unified diff between two items. If `strict` is true, will raise
-    an error if the items are of different formats.
-    """
-    if not from_item.body and not to_item.body:
-        raise ContentError(f"No body to diff for {from_item} and {to_item}")
-    if not from_item.store_path or not to_item.store_path:
-        raise ContentError("No store path on items; save before diffing")
-    diff_items = [item for item in [from_item, to_item] if item.format == Format.diff]
-    if len(diff_items) == 1:
-        raise ContentError(
-            f"Cannot compare diffs to non-diffs: {from_item.format}, {to_item.format}"
-        )
-    if len(diff_items) > 0 or from_item.format != to_item.format:
-        msg = f"Diffing items of incompatible format: {from_item.format}, {to_item.format}"
-        if strict:
-            raise ContentError(msg)
-        else:
-            log.warning("%s", msg)
-
-    from_path, to_path = StorePath(from_item.store_path), StorePath(to_item.store_path)
-
-    diff = unified_diff(from_item.body, to_item.body, str(from_path), str(to_path))
-
-    return Item(
-        type=ItemType.doc,
-        title=f"Diff of {from_path} and {to_path}",
-        format=Format.diff,
-        relations=ItemRelations(diff_of=[from_path, to_path]),
-        body=diff.patch_text,
-    )